mirror of
https://github.com/NVIDIA/cuda-samples.git
synced 2026-10-13 00:08:26 +08:00
CUDA 13.4 samples update - v13.4-public
Release 13.4 of the CUDA Samples supported by CUDA Toolkit 13.4. See Changelog for more information.
This commit is contained in:
Dheemanth
GitHub
@@ -1,18 +0,0 @@
|
||||
{
|
||||
"configurations": [
|
||||
{
|
||||
"name": "Linux",
|
||||
"includePath": [
|
||||
"${workspaceFolder}/**",
|
||||
"${workspaceFolder}/../../../Common"
|
||||
],
|
||||
"defines": [],
|
||||
"compilerPath": "/usr/local/cuda/bin/nvcc",
|
||||
"cStandard": "gnu17",
|
||||
"cppStandard": "gnu++14",
|
||||
"intelliSenseMode": "linux-gcc-x64",
|
||||
"configurationProvider": "ms-vscode.makefile-tools"
|
||||
}
|
||||
],
|
||||
"version": 4
|
||||
}
|
||||
@@ -1,7 +0,0 @@
|
||||
{
|
||||
"recommendations": [
|
||||
"nvidia.nsight-vscode-edition",
|
||||
"ms-vscode.cpptools",
|
||||
"ms-vscode.makefile-tools"
|
||||
]
|
||||
}
|
||||
@@ -1,34 +1,30 @@
|
||||
cmake_minimum_required(VERSION 3.20)
|
||||
|
||||
list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/../../../cmake/Modules")
|
||||
|
||||
include("${CMAKE_CURRENT_SOURCE_DIR}/../../../cmake/DetectCudaArch.cmake")
|
||||
|
||||
if(SAMPLE_SKIP_BUILD)
|
||||
return()
|
||||
endif()
|
||||
|
||||
project(simpleCallback LANGUAGES C CXX CUDA)
|
||||
|
||||
include("${CMAKE_CURRENT_SOURCE_DIR}/../../../cmake/CudaSampleCommon.cmake")
|
||||
|
||||
find_package(CUDAToolkit REQUIRED)
|
||||
|
||||
set(CMAKE_POSITION_INDEPENDENT_CODE ON)
|
||||
|
||||
set(CMAKE_CUDA_ARCHITECTURES 75 80 86 87 89 90 100 110 120)
|
||||
set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Wno-deprecated-gpu-targets")
|
||||
if(ENABLE_CUDA_DEBUG)
|
||||
set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -G") # enable cuda-gdb (may significantly affect performance on some targets)
|
||||
else()
|
||||
set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -lineinfo") # add line information to all builds for debug tools (exclusive to -G option)
|
||||
endif()
|
||||
|
||||
# Include directories and libraries
|
||||
include_directories(../../../Common)
|
||||
|
||||
# Source file
|
||||
# Add target for simpleCallback
|
||||
add_executable(simpleCallback simpleCallback.cu multithreading.cpp)
|
||||
add_executable(simpleCallback simpleCallback.cu)
|
||||
|
||||
target_compile_options(simpleCallback PRIVATE $<$<COMPILE_LANGUAGE:CUDA>:--extended-lambda>)
|
||||
|
||||
target_compile_features(simpleCallback PRIVATE cxx_std_17 cuda_std_17)
|
||||
|
||||
set_target_properties(simpleCallback PROPERTIES CUDA_SEPARABLE_COMPILATION ON)
|
||||
# std::thread is backed by pthreads on Linux
|
||||
find_package(Threads REQUIRED)
|
||||
target_link_libraries(simpleCallback PRIVATE Threads::Threads)
|
||||
|
||||
# Include installation configuration
|
||||
include(${CMAKE_CURRENT_SOURCE_DIR}/../../../cmake/InstallSamples.cmake)
|
||||
|
||||
include(InstallSamples)
|
||||
setup_samples_install()
|
||||
|
||||
@@ -1,32 +1,133 @@
|
||||
# simpleCallback - Simple CUDA Callbacks
|
||||
# Sample: simpleCallback
|
||||
|
||||
## Description
|
||||
|
||||
This sample implements multi-threaded heterogeneous computing workloads with the new CPU callbacks for CUDA streams and events introduced with CUDA 5.0.
|
||||
A heterogeneous pipeline demo: **CPU pre-process → GPU kernel → CPU post-process**, all
|
||||
coordinated by a single CUDA stream. A worker thread (`std::thread`) fills the input buffer,
|
||||
enqueues the GPU work (H2D copy, kernel, D2H copy), and registers a host function with
|
||||
`cudaLaunchHostFunc` — the modern replacement for the deprecated `cudaStreamAddCallback`.
|
||||
CUDA invokes that host function automatically once all preceding stream work completes,
|
||||
so the CPU post-processing (result verification) runs without the main thread ever polling.
|
||||
|
||||
## What You'll Learn
|
||||
|
||||
- Building a CPU → GPU → CPU pipeline ordered by a single CUDA stream
|
||||
- Scheduling CPU post-processing on a stream with `cudaLaunchHostFunc`
|
||||
- Enqueuing GPU work from a `std::thread` worker — all host threads share the device's primary context
|
||||
- The rule that a host function must never call CUDA APIs
|
||||
- Why pinned host memory (`cudaMallocHost`) is required for `cudaMemcpyAsync` to be truly asynchronous
|
||||
- Querying GPU properties (compute capability, SM count) with `cudaDeviceGetAttribute`
|
||||
|
||||
## Key Concepts
|
||||
|
||||
CUDA Streams, Callback Functions, Multithreading
|
||||
### Stream-Ordered Pipeline
|
||||
|
||||
## Supported SM Architectures
|
||||
A stream executes its operations strictly in order. The worker thread pushes the whole
|
||||
pipeline onto one stream and CUDA handles every dependency:
|
||||
|
||||
[SM 5.0 ](https://developer.nvidia.com/cuda-gpus) [SM 5.2 ](https://developer.nvidia.com/cuda-gpus) [SM 5.3 ](https://developer.nvidia.com/cuda-gpus) [SM 6.0 ](https://developer.nvidia.com/cuda-gpus) [SM 6.1 ](https://developer.nvidia.com/cuda-gpus) [SM 7.0 ](https://developer.nvidia.com/cuda-gpus) [SM 7.2 ](https://developer.nvidia.com/cuda-gpus) [SM 7.5 ](https://developer.nvidia.com/cuda-gpus) [SM 8.0 ](https://developer.nvidia.com/cuda-gpus) [SM 8.6 ](https://developer.nvidia.com/cuda-gpus) [SM 8.7 ](https://developer.nvidia.com/cuda-gpus) [SM 8.9 ](https://developer.nvidia.com/cuda-gpus) [SM 9.0 ](https://developer.nvidia.com/cuda-gpus)
|
||||
```text
|
||||
stream: [H2D copy] → [incrementKernel] → [D2H copy] → [postprocess host func]
|
||||
```
|
||||
|
||||
## Supported OSes
|
||||
The main thread only has to `cudaStreamSynchronize` once at the end — when the stream
|
||||
drains, the results are already verified.
|
||||
|
||||
Linux, Windows
|
||||
### Host Functions (`cudaLaunchHostFunc`)
|
||||
|
||||
## Supported CPU Architecture
|
||||
A host function is a CPU callback enqueued on a stream like any other operation. CUDA runs
|
||||
it on an internal thread once all prior work in the stream has finished. Two rules apply:
|
||||
|
||||
x86_64, armv7l
|
||||
- It must **not** call any CUDA runtime or driver API (no `cudaFree`, no kernel launches)
|
||||
- It should be short — the stream cannot proceed until it returns
|
||||
|
||||
## CUDA APIs involved
|
||||
### Threads Share the Primary Context
|
||||
|
||||
### [CUDA Runtime API](http://docs.nvidia.com/cuda/cuda-runtime-api/index.html)
|
||||
cudaHostAlloc, cudaStreamDestroy, cudaFree, cudaSetDevice, cudaGetDeviceCount, cudaFreeHost, cudaStreamAddCallback, cudaMalloc, cudaMemcpyAsync, cudaStreamCreate, cudaGetDeviceProperties
|
||||
With the CUDA runtime API, every host thread in the process shares the device's primary
|
||||
context. That is why the worker thread can enqueue copies and kernels onto a stream that
|
||||
`main` created.
|
||||
|
||||
## Prerequisites
|
||||
### Pinned Host Memory
|
||||
|
||||
Download and install the [CUDA Toolkit](https://developer.nvidia.com/cuda-downloads) for your corresponding platform.
|
||||
`cudaMemcpyAsync` only runs *truly asynchronously* when the host buffer is page-locked (pinned),
|
||||
so the OS cannot move the pages mid-transfer; with ordinary pageable memory the copy falls back to
|
||||
synchronous behavior. The sample allocates the host buffer with `cudaMallocHost`.
|
||||
|
||||
## References (for more details)
|
||||
## Key APIs
|
||||
|
||||
### CUDA Runtime
|
||||
|
||||
- `cudaSetDevice` — select the active GPU for all subsequent CUDA calls
|
||||
- `cudaDeviceGetAttribute` — query compute capability and SM count
|
||||
- `cudaMallocHost` / `cudaFreeHost` — allocate and free pinned host memory
|
||||
- `cudaMalloc` / `cudaFree` — allocate and free device memory
|
||||
- `cudaStreamCreate` / `cudaStreamDestroy` — create and destroy the stream that orders the pipeline
|
||||
- `cudaMemcpyAsync` — non-blocking H2D and D2H transfers on the stream
|
||||
- `cudaLaunchHostFunc` — enqueue a CPU callback on the stream
|
||||
- `cudaStreamSynchronize` — block the host until the stream (including the host function) is done
|
||||
|
||||
### C++ Standard Library
|
||||
|
||||
- `std::thread` / `join()` — worker-thread creation
|
||||
|
||||
## Requirements
|
||||
|
||||
### Hardware
|
||||
|
||||
- NVIDIA GPU with Compute Capability 7.5 or higher
|
||||
|
||||
### Software
|
||||
|
||||
- CMake 3.20 or newer
|
||||
- A C++17-capable host compiler
|
||||
|
||||
## How to Build
|
||||
|
||||
See the [top-level README](../../../README.md#building-cuda-samples) for full build instructions, including how to build all samples or a single sample standalone.
|
||||
|
||||
## How to Run
|
||||
|
||||
```bash
|
||||
./simpleCallback
|
||||
```
|
||||
|
||||
No command-line arguments are required. The sample always runs on device 0.
|
||||
|
||||
## Expected Output
|
||||
|
||||
```text
|
||||
=====================================================
|
||||
simpleCallback: CPU -> GPU -> CPU pipeline demo
|
||||
=====================================================
|
||||
Using GPU 0: compute capability X.Y, <smCount> SMs
|
||||
[thread] Stage 1: CPU pre-processing on a worker thread...
|
||||
[thread] Filled 100000 elements. First 3 inputs: 42, 43, 44
|
||||
|
||||
[thread] Stage 2: enqueuing H2D copy, kernel, D2H copy on the stream
|
||||
[thread] Registering post-processing callback with cudaLaunchHostFunc
|
||||
|
||||
[main] Worker thread joined; GPU work has been enqueued.
|
||||
[main] Waiting for the stream (and callback) to finish...
|
||||
|
||||
[host func] Stage 3: callback fired automatically - the GPU work is done!
|
||||
[host func] First 3 results: 43, 44, 45 (each input +1)
|
||||
[host func] Verified all 100000 results: PASS
|
||||
|
||||
[main] Pipeline complete. Result: SUCCESS
|
||||
```
|
||||
|
||||
**Reading the output:**
|
||||
- Inputs start at 42 because each element is `workload.id + i` with `id = 42`
|
||||
- Each result is its input plus one — the kernel's only job
|
||||
- The `[host func]` lines print from a CUDA internal thread, not from `main`
|
||||
|
||||
## Files
|
||||
|
||||
- `simpleCallback.cu` — kernel, worker-thread function, host-function callback, and main driver
|
||||
- `README.md` — this file
|
||||
- `CMakeLists.txt` — build configuration
|
||||
|
||||
## See Also
|
||||
|
||||
- [CUDA Programming Guide — Callback Functions from Streams](https://docs.nvidia.com/cuda/cuda-programming-guide/02-basics/asynchronous-execution.html#callback-functions-from-streams)
|
||||
- [CUDA Programming Guide — CUDA Streams](https://docs.nvidia.com/cuda/cuda-programming-guide/02-basics/asynchronous-execution.html#cuda-streams)
|
||||
- [CUDA Runtime API — Execution Control (cudaLaunchHostFunc)](https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__EXECUTION.html#group__CUDART__EXECUTION_1g05841eaa5f90f27124241baafb3e856f)
|
||||
|
||||
@@ -1,152 +0,0 @@
|
||||
/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions
|
||||
* are met:
|
||||
* * Redistributions of source code must retain the above copyright
|
||||
* notice, this list of conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in the
|
||||
* documentation and/or other materials provided with the distribution.
|
||||
* * Neither the name of NVIDIA CORPORATION nor the names of its
|
||||
* contributors may be used to endorse or promote products derived
|
||||
* from this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
|
||||
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
||||
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
|
||||
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
|
||||
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
|
||||
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
|
||||
* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
|
||||
* OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#include "multithreading.h"
|
||||
|
||||
#if defined(WIN32) || defined(_WIN32) || defined(WIN64) || defined(_WIN64)
|
||||
// Create thread
|
||||
CUTThread cutStartThread(CUT_THREADROUTINE func, void *data)
|
||||
{
|
||||
return CreateThread(NULL, 0, (LPTHREAD_START_ROUTINE)func, data, 0, NULL);
|
||||
}
|
||||
|
||||
// Wait for thread to finish
|
||||
void cutEndThread(CUTThread thread)
|
||||
{
|
||||
WaitForSingleObject(thread, INFINITE);
|
||||
CloseHandle(thread);
|
||||
}
|
||||
|
||||
// Wait for multiple threads
|
||||
void cutWaitForThreads(const CUTThread *threads, int num)
|
||||
{
|
||||
WaitForMultipleObjects(num, threads, true, INFINITE);
|
||||
|
||||
for (int i = 0; i < num; i++) {
|
||||
CloseHandle(threads[i]);
|
||||
}
|
||||
}
|
||||
|
||||
// Create barrier.
|
||||
CUTBarrier cutCreateBarrier(int releaseCount)
|
||||
{
|
||||
CUTBarrier barrier;
|
||||
|
||||
InitializeCriticalSection(&barrier.criticalSection);
|
||||
barrier.barrierEvent = CreateEvent(NULL, TRUE, FALSE, TEXT("BarrierEvent"));
|
||||
barrier.count = 0;
|
||||
barrier.releaseCount = releaseCount;
|
||||
|
||||
return barrier;
|
||||
}
|
||||
|
||||
// Increment barrier. (execution continues)
|
||||
void cutIncrementBarrier(CUTBarrier *barrier)
|
||||
{
|
||||
int myBarrierCount;
|
||||
EnterCriticalSection(&barrier->criticalSection);
|
||||
myBarrierCount = ++barrier->count;
|
||||
LeaveCriticalSection(&barrier->criticalSection);
|
||||
|
||||
if (myBarrierCount >= barrier->releaseCount) {
|
||||
SetEvent(barrier->barrierEvent);
|
||||
}
|
||||
}
|
||||
|
||||
// Wait for barrier release.
|
||||
void cutWaitForBarrier(CUTBarrier *barrier) { WaitForSingleObject(barrier->barrierEvent, INFINITE); }
|
||||
|
||||
// Destroy barrier
|
||||
void cutDestroyBarrier(CUTBarrier *barrier) {}
|
||||
|
||||
#else
|
||||
// Create thread
|
||||
CUTThread cutStartThread(CUT_THREADROUTINE func, void *data)
|
||||
{
|
||||
pthread_t thread;
|
||||
pthread_create(&thread, NULL, func, data);
|
||||
return thread;
|
||||
}
|
||||
|
||||
// Wait for thread to finish
|
||||
void cutEndThread(CUTThread thread) { pthread_join(thread, NULL); }
|
||||
|
||||
// Wait for multiple threads
|
||||
void cutWaitForThreads(const CUTThread *threads, int num)
|
||||
{
|
||||
for (int i = 0; i < num; i++) {
|
||||
cutEndThread(threads[i]);
|
||||
}
|
||||
}
|
||||
|
||||
// Create barrier.
|
||||
CUTBarrier cutCreateBarrier(int releaseCount)
|
||||
{
|
||||
CUTBarrier barrier;
|
||||
|
||||
barrier.count = 0;
|
||||
barrier.releaseCount = releaseCount;
|
||||
|
||||
pthread_mutex_init(&barrier.mutex, 0);
|
||||
pthread_cond_init(&barrier.conditionVariable, 0);
|
||||
|
||||
return barrier;
|
||||
}
|
||||
|
||||
// Increment barrier. (execution continues)
|
||||
void cutIncrementBarrier(CUTBarrier *barrier)
|
||||
{
|
||||
int myBarrierCount;
|
||||
pthread_mutex_lock(&barrier->mutex);
|
||||
myBarrierCount = ++barrier->count;
|
||||
pthread_mutex_unlock(&barrier->mutex);
|
||||
|
||||
if (myBarrierCount >= barrier->releaseCount) {
|
||||
pthread_cond_signal(&barrier->conditionVariable);
|
||||
}
|
||||
}
|
||||
|
||||
// Wait for barrier release.
|
||||
void cutWaitForBarrier(CUTBarrier *barrier)
|
||||
{
|
||||
pthread_mutex_lock(&barrier->mutex);
|
||||
|
||||
while (barrier->count < barrier->releaseCount) {
|
||||
pthread_cond_wait(&barrier->conditionVariable, &barrier->mutex);
|
||||
}
|
||||
|
||||
pthread_mutex_unlock(&barrier->mutex);
|
||||
}
|
||||
|
||||
// Destroy barrier
|
||||
void cutDestroyBarrier(CUTBarrier *barrier)
|
||||
{
|
||||
pthread_mutex_destroy(&barrier->mutex);
|
||||
pthread_cond_destroy(&barrier->conditionVariable);
|
||||
}
|
||||
|
||||
#endif
|
||||
@@ -1,101 +0,0 @@
|
||||
/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions
|
||||
* are met:
|
||||
* * Redistributions of source code must retain the above copyright
|
||||
* notice, this list of conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in the
|
||||
* documentation and/or other materials provided with the distribution.
|
||||
* * Neither the name of NVIDIA CORPORATION nor the names of its
|
||||
* contributors may be used to endorse or promote products derived
|
||||
* from this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
|
||||
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
||||
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
|
||||
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
|
||||
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
|
||||
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
|
||||
* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
|
||||
* OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#ifndef MULTITHREADING_H
|
||||
#define MULTITHREADING_H
|
||||
|
||||
// Simple portable thread library.
|
||||
|
||||
#if defined(WIN32) || defined(_WIN32) || defined(WIN64) || defined(_WIN64)
|
||||
// Windows threads.
|
||||
#include <windows.h>
|
||||
|
||||
typedef HANDLE CUTThread;
|
||||
typedef unsigned(WINAPI *CUT_THREADROUTINE)(void *);
|
||||
|
||||
struct CUTBarrier
|
||||
{
|
||||
CRITICAL_SECTION criticalSection;
|
||||
HANDLE barrierEvent;
|
||||
int releaseCount;
|
||||
int count;
|
||||
};
|
||||
|
||||
#define CUT_THREADPROC unsigned WINAPI
|
||||
#define CUT_THREADEND return 0
|
||||
|
||||
#else
|
||||
// POSIX threads.
|
||||
#include <pthread.h>
|
||||
|
||||
typedef pthread_t CUTThread;
|
||||
typedef void *(*CUT_THREADROUTINE)(void *);
|
||||
|
||||
#define CUT_THREADPROC void *
|
||||
#define CUT_THREADEND return 0
|
||||
|
||||
struct CUTBarrier
|
||||
{
|
||||
pthread_mutex_t mutex;
|
||||
pthread_cond_t conditionVariable;
|
||||
int releaseCount;
|
||||
int count;
|
||||
};
|
||||
|
||||
#endif
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C"
|
||||
{
|
||||
#endif
|
||||
|
||||
// Create thread.
|
||||
CUTThread cutStartThread(CUT_THREADROUTINE, void *data);
|
||||
|
||||
// Wait for thread to finish.
|
||||
void cutEndThread(CUTThread thread);
|
||||
|
||||
// Wait for multiple threads.
|
||||
void cutWaitForThreads(const CUTThread *threads, int num);
|
||||
|
||||
// Create barrier.
|
||||
CUTBarrier cutCreateBarrier(int releaseCount);
|
||||
|
||||
// Increment barrier. (execution continues)
|
||||
void cutIncrementBarrier(CUTBarrier *barrier);
|
||||
|
||||
// Wait for barrier release.
|
||||
void cutWaitForBarrier(CUTBarrier *barrier);
|
||||
|
||||
// Destroy barrier
|
||||
void cutDestroyBarrier(CUTBarrier *barrier);
|
||||
|
||||
#ifdef __cplusplus
|
||||
} // extern "C"
|
||||
#endif
|
||||
|
||||
#endif // MULTITHREADING_H
|
||||
@@ -1,4 +1,4 @@
|
||||
/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
|
||||
/* Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions
|
||||
@@ -26,190 +26,171 @@
|
||||
*/
|
||||
|
||||
/*
|
||||
* This sample implements multi-threaded heterogeneous computing workloads with
|
||||
* the new CPU callbacks for CUDA streams and events introduced with CUDA 5.0.
|
||||
* Together with the thread safety of the CUDA API implementing heterogeneous
|
||||
* workloads that float between CPU threads and GPUs has become simple and
|
||||
* efficient.
|
||||
* simpleCallback
|
||||
* --------------
|
||||
* An example of a heterogeneous pipeline in the form:
|
||||
*
|
||||
* The workloads in the sample follow the form CPU preprocess -> GPU process ->
|
||||
* CPU postprocess.
|
||||
* Each CPU processing step is handled by its own dedicated thread. GPU
|
||||
* workloads are sent to all available GPUs in the system.
|
||||
* CPU pre-process -> GPU kernel -> CPU post-process
|
||||
*
|
||||
* The whole pipeline is coordinated by a *single* CUDA stream. A stream
|
||||
* executes the operations enqueued on it in order, so we can simply push work
|
||||
* onto it and let CUDA take care of ordering and dependencies for us.
|
||||
*
|
||||
* Two different "CPU" mechanisms appear in this sample so you can see how they
|
||||
* differ:
|
||||
*
|
||||
* 1. The CPU pre-processing runs on a dedicated worker thread created with
|
||||
* C++ std::thread. This shows how ordinary host-side threading can be
|
||||
* combined with CUDA. The worker fills the input buffer and then enqueues
|
||||
* the GPU pipeline itself; main joins it and waits on the stream.
|
||||
*
|
||||
* 2. The CPU post-processing is scheduled *on the stream itself* with
|
||||
* cudaLaunchHostFunc(). CUDA calls our host function automatically once all
|
||||
* the preceding stream work (copies + kernel) has completed. This is the
|
||||
* modern, non-deprecated replacement for cudaStreamAddCallback().
|
||||
*
|
||||
* IMPORTANT rule for cudaLaunchHostFunc callbacks:
|
||||
* The host function must NOT call any CUDA runtime/driver API (no cudaMalloc,
|
||||
* cudaFree, kernel launches, etc.). It should only do plain CPU work.
|
||||
*
|
||||
* NOTE: For clarity this sample intentionally omits CUDA error checking. Real
|
||||
* applications should check the return code of every CUDA call.
|
||||
*/
|
||||
|
||||
// System includes
|
||||
#include <stdio.h>
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
|
||||
// helper functions and utilities to work with CUDA
|
||||
#include <helper_cuda.h>
|
||||
#include <helper_functions.h>
|
||||
#include <thread> // std::thread — C++11 standard threading
|
||||
|
||||
#include "multithreading.h"
|
||||
// CUDA runtime
|
||||
#include <cuda_runtime.h>
|
||||
|
||||
const int N_workloads = 8;
|
||||
const int N_elements_per_workload = 100000;
|
||||
const int NumElements = 100000;
|
||||
|
||||
CUTBarrier thread_barrier;
|
||||
|
||||
void CUDART_CB myStreamCallback(cudaStream_t event, cudaError_t status, void *data);
|
||||
|
||||
struct heterogeneous_workload
|
||||
// Everything a single pipeline needs. Passed by pointer to both the
|
||||
// std::thread worker and to the stream host-function callback.
|
||||
struct Workload
|
||||
{
|
||||
int id;
|
||||
int cudaDeviceID;
|
||||
|
||||
int *h_data;
|
||||
int *d_data;
|
||||
cudaStream_t stream;
|
||||
|
||||
bool success;
|
||||
int id = 0; // arbitrary tag added to each element
|
||||
int *h_data = nullptr; // pinned host buffer (fast async copies)
|
||||
int *d_data = nullptr; // device buffer
|
||||
cudaStream_t stream = nullptr; // the single stream that orders all the work
|
||||
bool success = false; // set by the post-processing callback
|
||||
};
|
||||
|
||||
__global__ void incKernel(int *data, int N)
|
||||
// GPU kernel: increment every element by one.
|
||||
__global__ void incrementKernel(int *data, int n)
|
||||
{
|
||||
int i = blockIdx.x * blockDim.x + threadIdx.x;
|
||||
|
||||
if (i < N)
|
||||
data[i]++;
|
||||
if (i < n) {
|
||||
data[i] += 1;
|
||||
}
|
||||
}
|
||||
|
||||
CUT_THREADPROC launch(void *void_arg)
|
||||
// Forward declaration: postprocess is defined below but called from preprocess.
|
||||
void CUDART_CB postprocess(void *arg);
|
||||
|
||||
// Worker thread: CPU pre-processing + full GPU pipeline enqueue.
|
||||
//
|
||||
// std::thread passes arguments with their actual types, so no void* casting
|
||||
// is needed. This thread fills the input buffer, then enqueues the full GPU
|
||||
// pipeline (H2D, kernel, D2H, host func) onto the workload's stream.
|
||||
void preprocess(Workload *workload)
|
||||
{
|
||||
heterogeneous_workload *workload = (heterogeneous_workload *)void_arg;
|
||||
|
||||
// Select GPU for this CPU thread
|
||||
checkCudaErrors(cudaSetDevice(workload->cudaDeviceID));
|
||||
|
||||
// Allocate Resources
|
||||
checkCudaErrors(cudaStreamCreate(&workload->stream));
|
||||
checkCudaErrors(cudaMalloc(&workload->d_data, N_elements_per_workload * sizeof(int)));
|
||||
checkCudaErrors(cudaHostAlloc(&workload->h_data, N_elements_per_workload * sizeof(int), cudaHostAllocPortable));
|
||||
|
||||
// CPU thread generates data
|
||||
for (int i = 0; i < N_elements_per_workload; ++i) {
|
||||
// Stage 1: CPU pre-processing ----------------------------------
|
||||
printf("[thread] Stage 1: CPU pre-processing on a worker thread...\n");
|
||||
for (int i = 0; i < NumElements; ++i) {
|
||||
workload->h_data[i] = workload->id + i;
|
||||
}
|
||||
printf("[thread] Filled %d elements. First 3 inputs: %d, %d, %d\n\n",
|
||||
NumElements, workload->h_data[0], workload->h_data[1], workload->h_data[2]);
|
||||
|
||||
// Schedule work for GPU in CUDA stream without blocking the CPU thread
|
||||
// Note: Dedicated streams enable concurrent execution of workloads on the GPU
|
||||
dim3 block(512);
|
||||
dim3 grid((N_elements_per_workload + block.x - 1) / block.x);
|
||||
// Stage 2: enqueue GPU work from this thread -------------------
|
||||
const size_t bytes = NumElements * sizeof(int);
|
||||
const int threadsPerBlock = 256;
|
||||
const int blocks = (NumElements + threadsPerBlock - 1) / threadsPerBlock;
|
||||
|
||||
checkCudaErrors(cudaMemcpyAsync(workload->d_data,
|
||||
workload->h_data,
|
||||
N_elements_per_workload * sizeof(int),
|
||||
cudaMemcpyHostToDevice,
|
||||
workload->stream));
|
||||
incKernel<<<grid, block, 0, workload->stream>>>(workload->d_data, N_elements_per_workload);
|
||||
checkCudaErrors(cudaMemcpyAsync(workload->h_data,
|
||||
workload->d_data,
|
||||
N_elements_per_workload * sizeof(int),
|
||||
cudaMemcpyDeviceToHost,
|
||||
workload->stream));
|
||||
printf("[thread] Stage 2: enqueuing H2D copy, kernel, D2H copy on the stream\n");
|
||||
cudaMemcpyAsync(workload->d_data, workload->h_data, bytes, cudaMemcpyHostToDevice, workload->stream);
|
||||
incrementKernel<<<blocks, threadsPerBlock, 0, workload->stream>>>(workload->d_data, NumElements);
|
||||
cudaMemcpyAsync(workload->h_data, workload->d_data, bytes, cudaMemcpyDeviceToHost, workload->stream);
|
||||
|
||||
// New in CUDA 5.0: Add a CPU callback which is called once all currently
|
||||
// pending operations in the CUDA stream have finished
|
||||
checkCudaErrors(cudaStreamAddCallback(workload->stream, myStreamCallback, workload, 0));
|
||||
|
||||
CUT_THREADEND;
|
||||
// CPU thread end of life, GPU continues to process data...
|
||||
// Stage 3: schedule CPU post-processing on the stream -----------
|
||||
printf("[thread] Registering post-processing callback with cudaLaunchHostFunc\n\n");
|
||||
cudaLaunchHostFunc(workload->stream, postprocess, workload);
|
||||
}
|
||||
|
||||
CUT_THREADPROC postprocess(void *void_arg)
|
||||
// Stage 3: CPU post-processing, scheduled on the stream via cudaLaunchHostFunc.
|
||||
//
|
||||
// CUDA invokes this automatically once the H2D copy, the kernel, and the D2H
|
||||
// copy that precede it on the stream have all finished. Every element should
|
||||
// now equal its original value plus one (from the kernel).
|
||||
//
|
||||
// Reminder: DO NOT call any CUDA API from inside this function.
|
||||
void CUDART_CB postprocess(void *arg)
|
||||
{
|
||||
heterogeneous_workload *workload = (heterogeneous_workload *)void_arg;
|
||||
// ... GPU is done with processing, continue on new CPU thread...
|
||||
// recover the typed pointer from the callback's void* arg
|
||||
auto *workload = static_cast<Workload *>(arg);
|
||||
|
||||
// Select GPU for this CPU thread
|
||||
checkCudaErrors(cudaSetDevice(workload->cudaDeviceID));
|
||||
// Stage 3: CPU post-processing — runs automatically when all stream work above is done
|
||||
printf("[host func] Stage 3: callback fired automatically - the GPU work is done!\n");
|
||||
printf("[host func] First 3 results: %d, %d, %d (each input +1)\n",
|
||||
workload->h_data[0], workload->h_data[1], workload->h_data[2]);
|
||||
|
||||
// CPU thread consumes results from GPU
|
||||
workload->success = true;
|
||||
|
||||
for (int i = 0; i < N_workloads; ++i) {
|
||||
workload->success &= workload->h_data[i] == i + workload->id + 1;
|
||||
}
|
||||
|
||||
// Free Resources
|
||||
checkCudaErrors(cudaFree(workload->d_data));
|
||||
checkCudaErrors(cudaFreeHost(workload->h_data));
|
||||
checkCudaErrors(cudaStreamDestroy(workload->stream));
|
||||
|
||||
// Signal the end of the heterogeneous workload to main thread
|
||||
cutIncrementBarrier(&thread_barrier);
|
||||
|
||||
CUT_THREADEND;
|
||||
}
|
||||
|
||||
void CUDART_CB myStreamCallback(cudaStream_t stream, cudaError_t status, void *data)
|
||||
{
|
||||
// Check status of GPU after stream operations are done
|
||||
checkCudaErrors(status);
|
||||
|
||||
// Spawn new CPU worker thread and continue processing on the CPU
|
||||
cutStartThread(postprocess, data);
|
||||
}
|
||||
|
||||
int main(int argc, char **argv)
|
||||
{
|
||||
int N_gpus, max_gpus = 0;
|
||||
int gpuInfo[32]; // assume a maximum of 32 GPUs in a system configuration
|
||||
|
||||
printf("Starting simpleCallback\n");
|
||||
|
||||
checkCudaErrors(cudaGetDeviceCount(&N_gpus));
|
||||
printf("Found %d CUDA capable GPUs\n", N_gpus);
|
||||
|
||||
if (N_gpus > 32) {
|
||||
printf("simpleCallback only supports 32 GPU(s)\n");
|
||||
}
|
||||
|
||||
for (int devid = 0; devid < N_gpus; devid++) {
|
||||
int SMversion;
|
||||
cudaDeviceProp deviceProp;
|
||||
cudaSetDevice(devid);
|
||||
cudaGetDeviceProperties(&deviceProp, devid);
|
||||
SMversion = deviceProp.major << 4 + deviceProp.minor;
|
||||
printf("GPU[%d] %s supports SM %d.%d", devid, deviceProp.name, deviceProp.major, deviceProp.minor);
|
||||
printf(", %s GPU Callback Functions\n", (SMversion >= 0x11) ? "capable" : "NOT capable");
|
||||
|
||||
if (SMversion >= 0x11) {
|
||||
gpuInfo[max_gpus++] = devid;
|
||||
bool verify = true;
|
||||
for (int i = 0; i < NumElements; ++i) {
|
||||
if (workload->h_data[i] != workload->id + i + 1) {
|
||||
verify = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
printf("%d GPUs available to run Callback Functions\n", max_gpus);
|
||||
|
||||
heterogeneous_workload *workloads;
|
||||
workloads = (heterogeneous_workload *)malloc(N_workloads * sizeof(heterogeneous_workload));
|
||||
;
|
||||
thread_barrier = cutCreateBarrier(N_workloads);
|
||||
|
||||
// Main thread spawns a CPU worker thread for each heterogeneous workload
|
||||
printf("Starting %d heterogeneous computing workloads\n", N_workloads);
|
||||
|
||||
for (int i = 0; i < N_workloads; ++i) {
|
||||
workloads[i].id = i;
|
||||
workloads[i].cudaDeviceID = gpuInfo[i % max_gpus]; // i % N_gpus;
|
||||
|
||||
cutStartThread(launch, &workloads[i]);
|
||||
}
|
||||
|
||||
// Sleep until all workloads have finished
|
||||
cutWaitForBarrier(&thread_barrier);
|
||||
printf("Total of %d workloads finished:\n", N_workloads);
|
||||
|
||||
bool success = true;
|
||||
|
||||
for (int i = 0; i < N_workloads; ++i) {
|
||||
success &= workloads[i].success;
|
||||
}
|
||||
|
||||
printf("%s\n", success ? "Success" : "Failure");
|
||||
|
||||
free(workloads);
|
||||
|
||||
exit(success ? EXIT_SUCCESS : EXIT_FAILURE);
|
||||
workload->success = verify;
|
||||
printf("[host func] Verified all %d results: %s\n", NumElements, verify ? "PASS" : "MISMATCH");
|
||||
}
|
||||
|
||||
int main()
|
||||
{
|
||||
printf("=====================================================\n");
|
||||
printf(" simpleCallback: CPU -> GPU -> CPU pipeline demo\n");
|
||||
printf("=====================================================\n");
|
||||
|
||||
// Use the first available CUDA device.
|
||||
int devID = 0;
|
||||
cudaSetDevice(devID);
|
||||
|
||||
int major = 0, minor = 0, smCount = 0;
|
||||
cudaDeviceGetAttribute(&major, cudaDevAttrComputeCapabilityMajor, devID);
|
||||
cudaDeviceGetAttribute(&minor, cudaDevAttrComputeCapabilityMinor, devID);
|
||||
cudaDeviceGetAttribute(&smCount, cudaDevAttrMultiProcessorCount, devID);
|
||||
printf("Using GPU %d: compute capability %d.%d, %d SMs\n", devID, major, minor, smCount);
|
||||
|
||||
// Allocate resources
|
||||
Workload workload;
|
||||
workload.id = 42; // Any number can be used
|
||||
|
||||
// Pinned (page-locked) host memory enables true asynchronous copies.
|
||||
const size_t bytes = NumElements * sizeof(int);
|
||||
cudaMallocHost(&workload.h_data, bytes);
|
||||
cudaMalloc(&workload.d_data, bytes);
|
||||
cudaStreamCreate(&workload.stream);
|
||||
|
||||
// Launch worker thread; it fills the buffer AND enqueues the full GPU pipeline.
|
||||
std::thread worker(preprocess, &workload);
|
||||
worker.join();
|
||||
printf("[main] Worker thread joined; GPU work has been enqueued.\n");
|
||||
|
||||
// Block until the whole pipeline (including the host function) is complete.
|
||||
printf("[main] Waiting for the stream (and callback) to finish...\n\n");
|
||||
cudaStreamSynchronize(workload.stream);
|
||||
|
||||
// Clean up
|
||||
// Safe to call CUDA APIs here: we are back on the main thread, not inside
|
||||
// the host-function callback.
|
||||
cudaStreamDestroy(workload.stream);
|
||||
cudaFree(workload.d_data);
|
||||
cudaFreeHost(workload.h_data);
|
||||
|
||||
printf("\n[main] Pipeline complete. Result: %s\n", workload.success ? "SUCCESS" : "FAILURE");
|
||||
return workload.success ? EXIT_SUCCESS : EXIT_FAILURE;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user