CUDA 13.2 samples update (#432)

- Added Python samples for CUDA Python 1.0 release
- Renamed top-level `Samples` directory to `cpp` to accommodate Python samples.

cpp/0_Introduction/simpleCallback/.vscode/c_cpp_properties.json

@@ -0,0 +1,18 @@
+{
+    "configurations": [
+        {
+            "name": "Linux",
+            "includePath": [
+                "${workspaceFolder}/**",
+                "${workspaceFolder}/../../../Common"
+            ],
+            "defines": [],
+            "compilerPath": "/usr/local/cuda/bin/nvcc",
+            "cStandard": "gnu17",
+            "cppStandard": "gnu++14",
+            "intelliSenseMode": "linux-gcc-x64",
+            "configurationProvider": "ms-vscode.makefile-tools"
+        }
+    ],
+    "version": 4
+}

cpp/0_Introduction/simpleCallback/.vscode/extensions.json

@@ -0,0 +1,7 @@
+{
+    "recommendations": [
+        "nvidia.nsight-vscode-edition",
+        "ms-vscode.cpptools",
+        "ms-vscode.makefile-tools"
+    ]
+}

cpp/0_Introduction/simpleCallback/CMakeLists.txt

@@ -0,0 +1,34 @@
+cmake_minimum_required(VERSION 3.20)
+
+list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/../../../cmake/Modules")
+
+project(simpleCallback LANGUAGES C CXX CUDA)
+
+find_package(CUDAToolkit REQUIRED)
+
+set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+
+set(CMAKE_CUDA_ARCHITECTURES 75 80 86 87 89 90 100 110 120)
+set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Wno-deprecated-gpu-targets")
+if(ENABLE_CUDA_DEBUG)
+    set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -G")        # enable cuda-gdb (may significantly affect performance on some targets)
+else()
+    set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -lineinfo") # add line information to all builds for debug tools (exclusive to -G option)
+endif()
+
+# Include directories and libraries
+include_directories(../../../Common)
+
+# Source file
+# Add target for simpleCallback
+add_executable(simpleCallback simpleCallback.cu multithreading.cpp)
+
+target_compile_options(simpleCallback PRIVATE $<$<COMPILE_LANGUAGE:CUDA>:--extended-lambda>)
+
+target_compile_features(simpleCallback PRIVATE cxx_std_17 cuda_std_17)
+
+set_target_properties(simpleCallback PROPERTIES CUDA_SEPARABLE_COMPILATION ON)
+
+# Include installation configuration
+include(${CMAKE_CURRENT_SOURCE_DIR}/../../../cmake/InstallSamples.cmake)
+setup_samples_install()

cpp/0_Introduction/simpleCallback/README.md

@@ -0,0 +1,32 @@
+# simpleCallback - Simple CUDA Callbacks
+
+## Description
+
+This sample implements multi-threaded heterogeneous computing workloads with the new CPU callbacks for CUDA streams and events introduced with CUDA 5.0.
+
+## Key Concepts
+
+CUDA Streams, Callback Functions, Multithreading
+
+## Supported SM Architectures
+
+[SM 5.0 ](https://developer.nvidia.com/cuda-gpus)  [SM 5.2 ](https://developer.nvidia.com/cuda-gpus)  [SM 5.3 ](https://developer.nvidia.com/cuda-gpus)  [SM 6.0 ](https://developer.nvidia.com/cuda-gpus)  [SM 6.1 ](https://developer.nvidia.com/cuda-gpus)  [SM 7.0 ](https://developer.nvidia.com/cuda-gpus)  [SM 7.2 ](https://developer.nvidia.com/cuda-gpus)  [SM 7.5 ](https://developer.nvidia.com/cuda-gpus)  [SM 8.0 ](https://developer.nvidia.com/cuda-gpus)  [SM 8.6 ](https://developer.nvidia.com/cuda-gpus)  [SM 8.7 ](https://developer.nvidia.com/cuda-gpus)  [SM 8.9 ](https://developer.nvidia.com/cuda-gpus)  [SM 9.0 ](https://developer.nvidia.com/cuda-gpus)
+
+## Supported OSes
+
+Linux, Windows
+
+## Supported CPU Architecture
+
+x86_64, armv7l
+
+## CUDA APIs involved
+
+### [CUDA Runtime API](http://docs.nvidia.com/cuda/cuda-runtime-api/index.html)
+cudaHostAlloc, cudaStreamDestroy, cudaFree, cudaSetDevice, cudaGetDeviceCount, cudaFreeHost, cudaStreamAddCallback, cudaMalloc, cudaMemcpyAsync, cudaStreamCreate, cudaGetDeviceProperties
+
+## Prerequisites
+
+Download and install the [CUDA Toolkit](https://developer.nvidia.com/cuda-downloads) for your corresponding platform.
+
+## References (for more details)

cpp/0_Introduction/simpleCallback/multithreading.cpp

@@ -0,0 +1,152 @@
+/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "multithreading.h"
+
+#if defined(WIN32) || defined(_WIN32) || defined(WIN64) || defined(_WIN64)
+// Create thread
+CUTThread cutStartThread(CUT_THREADROUTINE func, void *data)
+{
+    return CreateThread(NULL, 0, (LPTHREAD_START_ROUTINE)func, data, 0, NULL);
+}
+
+// Wait for thread to finish
+void cutEndThread(CUTThread thread)
+{
+    WaitForSingleObject(thread, INFINITE);
+    CloseHandle(thread);
+}
+
+// Wait for multiple threads
+void cutWaitForThreads(const CUTThread *threads, int num)
+{
+    WaitForMultipleObjects(num, threads, true, INFINITE);
+
+    for (int i = 0; i < num; i++) {
+        CloseHandle(threads[i]);
+    }
+}
+
+// Create barrier.
+CUTBarrier cutCreateBarrier(int releaseCount)
+{
+    CUTBarrier barrier;
+
+    InitializeCriticalSection(&barrier.criticalSection);
+    barrier.barrierEvent = CreateEvent(NULL, TRUE, FALSE, TEXT("BarrierEvent"));
+    barrier.count        = 0;
+    barrier.releaseCount = releaseCount;
+
+    return barrier;
+}
+
+// Increment barrier. (execution continues)
+void cutIncrementBarrier(CUTBarrier *barrier)
+{
+    int myBarrierCount;
+    EnterCriticalSection(&barrier->criticalSection);
+    myBarrierCount = ++barrier->count;
+    LeaveCriticalSection(&barrier->criticalSection);
+
+    if (myBarrierCount >= barrier->releaseCount) {
+        SetEvent(barrier->barrierEvent);
+    }
+}
+
+// Wait for barrier release.
+void cutWaitForBarrier(CUTBarrier *barrier) { WaitForSingleObject(barrier->barrierEvent, INFINITE); }
+
+// Destroy barrier
+void cutDestroyBarrier(CUTBarrier *barrier) {}
+
+#else
+// Create thread
+CUTThread cutStartThread(CUT_THREADROUTINE func, void *data)
+{
+    pthread_t thread;
+    pthread_create(&thread, NULL, func, data);
+    return thread;
+}
+
+// Wait for thread to finish
+void cutEndThread(CUTThread thread) { pthread_join(thread, NULL); }
+
+// Wait for multiple threads
+void cutWaitForThreads(const CUTThread *threads, int num)
+{
+    for (int i = 0; i < num; i++) {
+        cutEndThread(threads[i]);
+    }
+}
+
+// Create barrier.
+CUTBarrier cutCreateBarrier(int releaseCount)
+{
+    CUTBarrier barrier;
+
+    barrier.count        = 0;
+    barrier.releaseCount = releaseCount;
+
+    pthread_mutex_init(&barrier.mutex, 0);
+    pthread_cond_init(&barrier.conditionVariable, 0);
+
+    return barrier;
+}
+
+// Increment barrier. (execution continues)
+void cutIncrementBarrier(CUTBarrier *barrier)
+{
+    int myBarrierCount;
+    pthread_mutex_lock(&barrier->mutex);
+    myBarrierCount = ++barrier->count;
+    pthread_mutex_unlock(&barrier->mutex);
+
+    if (myBarrierCount >= barrier->releaseCount) {
+        pthread_cond_signal(&barrier->conditionVariable);
+    }
+}
+
+// Wait for barrier release.
+void cutWaitForBarrier(CUTBarrier *barrier)
+{
+    pthread_mutex_lock(&barrier->mutex);
+
+    while (barrier->count < barrier->releaseCount) {
+        pthread_cond_wait(&barrier->conditionVariable, &barrier->mutex);
+    }
+
+    pthread_mutex_unlock(&barrier->mutex);
+}
+
+// Destroy barrier
+void cutDestroyBarrier(CUTBarrier *barrier)
+{
+    pthread_mutex_destroy(&barrier->mutex);
+    pthread_cond_destroy(&barrier->conditionVariable);
+}
+
+#endif

cpp/0_Introduction/simpleCallback/multithreading.h

@@ -0,0 +1,101 @@
+/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef MULTITHREADING_H
+#define MULTITHREADING_H
+
+// Simple portable thread library.
+
+#if defined(WIN32) || defined(_WIN32) || defined(WIN64) || defined(_WIN64)
+// Windows threads.
+#include <windows.h>
+
+typedef HANDLE CUTThread;
+typedef unsigned(WINAPI *CUT_THREADROUTINE)(void *);
+
+struct CUTBarrier
+{
+    CRITICAL_SECTION criticalSection;
+    HANDLE           barrierEvent;
+    int              releaseCount;
+    int              count;
+};
+
+#define CUT_THREADPROC unsigned WINAPI
+#define CUT_THREADEND  return 0
+
+#else
+// POSIX threads.
+#include <pthread.h>
+
+typedef pthread_t CUTThread;
+typedef void *(*CUT_THREADROUTINE)(void *);
+
+#define CUT_THREADPROC void *
+#define CUT_THREADEND  return 0
+
+struct CUTBarrier
+{
+    pthread_mutex_t mutex;
+    pthread_cond_t  conditionVariable;
+    int             releaseCount;
+    int             count;
+};
+
+#endif
+
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+
+    // Create thread.
+    CUTThread cutStartThread(CUT_THREADROUTINE, void *data);
+
+    // Wait for thread to finish.
+    void cutEndThread(CUTThread thread);
+
+    // Wait for multiple threads.
+    void cutWaitForThreads(const CUTThread *threads, int num);
+
+    // Create barrier.
+    CUTBarrier cutCreateBarrier(int releaseCount);
+
+    // Increment barrier. (execution continues)
+    void cutIncrementBarrier(CUTBarrier *barrier);
+
+    // Wait for barrier release.
+    void cutWaitForBarrier(CUTBarrier *barrier);
+
+    // Destroy barrier
+    void cutDestroyBarrier(CUTBarrier *barrier);
+
+#ifdef __cplusplus
+} // extern "C"
+#endif
+
+#endif // MULTITHREADING_H

cpp/0_Introduction/simpleCallback/simpleCallback.cu

@@ -0,0 +1,215 @@
+/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * This sample implements multi-threaded heterogeneous computing workloads with
+ * the new CPU callbacks for CUDA streams and events introduced with CUDA 5.0.
+ * Together with the thread safety of the CUDA API implementing heterogeneous
+ * workloads that float between CPU threads and GPUs has become simple and
+ * efficient.
+ *
+ * The workloads in the sample follow the form CPU preprocess -> GPU process ->
+ * CPU postprocess.
+ * Each CPU processing step is handled by its own dedicated thread. GPU
+ * workloads are sent to all available GPUs in the system.
+ *
+ */
+
+// System includes
+#include <stdio.h>
+
+// helper functions and utilities to work with CUDA
+#include <helper_cuda.h>
+#include <helper_functions.h>
+
+#include "multithreading.h"
+
+const int N_workloads             = 8;
+const int N_elements_per_workload = 100000;
+
+CUTBarrier thread_barrier;
+
+void CUDART_CB myStreamCallback(cudaStream_t event, cudaError_t status, void *data);
+
+struct heterogeneous_workload
+{
+    int id;
+    int cudaDeviceID;
+
+    int         *h_data;
+    int         *d_data;
+    cudaStream_t stream;
+
+    bool success;
+};
+
+__global__ void incKernel(int *data, int N)
+{
+    int i = blockIdx.x * blockDim.x + threadIdx.x;
+
+    if (i < N)
+        data[i]++;
+}
+
+CUT_THREADPROC launch(void *void_arg)
+{
+    heterogeneous_workload *workload = (heterogeneous_workload *)void_arg;
+
+    // Select GPU for this CPU thread
+    checkCudaErrors(cudaSetDevice(workload->cudaDeviceID));
+
+    // Allocate Resources
+    checkCudaErrors(cudaStreamCreate(&workload->stream));
+    checkCudaErrors(cudaMalloc(&workload->d_data, N_elements_per_workload * sizeof(int)));
+    checkCudaErrors(cudaHostAlloc(&workload->h_data, N_elements_per_workload * sizeof(int), cudaHostAllocPortable));
+
+    // CPU thread generates data
+    for (int i = 0; i < N_elements_per_workload; ++i) {
+        workload->h_data[i] = workload->id + i;
+    }
+
+    // Schedule work for GPU in CUDA stream without blocking the CPU thread
+    // Note: Dedicated streams enable concurrent execution of workloads on the GPU
+    dim3 block(512);
+    dim3 grid((N_elements_per_workload + block.x - 1) / block.x);
+
+    checkCudaErrors(cudaMemcpyAsync(workload->d_data,
+                                    workload->h_data,
+                                    N_elements_per_workload * sizeof(int),
+                                    cudaMemcpyHostToDevice,
+                                    workload->stream));
+    incKernel<<<grid, block, 0, workload->stream>>>(workload->d_data, N_elements_per_workload);
+    checkCudaErrors(cudaMemcpyAsync(workload->h_data,
+                                    workload->d_data,
+                                    N_elements_per_workload * sizeof(int),
+                                    cudaMemcpyDeviceToHost,
+                                    workload->stream));
+
+    // New in CUDA 5.0: Add a CPU callback which is called once all currently
+    // pending operations in the CUDA stream have finished
+    checkCudaErrors(cudaStreamAddCallback(workload->stream, myStreamCallback, workload, 0));
+
+    CUT_THREADEND;
+    // CPU thread end of life, GPU continues to process data...
+}
+
+CUT_THREADPROC postprocess(void *void_arg)
+{
+    heterogeneous_workload *workload = (heterogeneous_workload *)void_arg;
+    // ... GPU is done with processing, continue on new CPU thread...
+
+    // Select GPU for this CPU thread
+    checkCudaErrors(cudaSetDevice(workload->cudaDeviceID));
+
+    // CPU thread consumes results from GPU
+    workload->success = true;
+
+    for (int i = 0; i < N_workloads; ++i) {
+        workload->success &= workload->h_data[i] == i + workload->id + 1;
+    }
+
+    // Free Resources
+    checkCudaErrors(cudaFree(workload->d_data));
+    checkCudaErrors(cudaFreeHost(workload->h_data));
+    checkCudaErrors(cudaStreamDestroy(workload->stream));
+
+    // Signal the end of the heterogeneous workload to main thread
+    cutIncrementBarrier(&thread_barrier);
+
+    CUT_THREADEND;
+}
+
+void CUDART_CB myStreamCallback(cudaStream_t stream, cudaError_t status, void *data)
+{
+    // Check status of GPU after stream operations are done
+    checkCudaErrors(status);
+
+    // Spawn new CPU worker thread and continue processing on the CPU
+    cutStartThread(postprocess, data);
+}
+
+int main(int argc, char **argv)
+{
+    int N_gpus, max_gpus = 0;
+    int gpuInfo[32]; // assume a maximum of 32 GPUs in a system configuration
+
+    printf("Starting simpleCallback\n");
+
+    checkCudaErrors(cudaGetDeviceCount(&N_gpus));
+    printf("Found %d CUDA capable GPUs\n", N_gpus);
+
+    if (N_gpus > 32) {
+        printf("simpleCallback only supports 32 GPU(s)\n");
+    }
+
+    for (int devid = 0; devid < N_gpus; devid++) {
+        int            SMversion;
+        cudaDeviceProp deviceProp;
+        cudaSetDevice(devid);
+        cudaGetDeviceProperties(&deviceProp, devid);
+        SMversion = deviceProp.major << 4 + deviceProp.minor;
+        printf("GPU[%d] %s supports SM %d.%d", devid, deviceProp.name, deviceProp.major, deviceProp.minor);
+        printf(", %s GPU Callback Functions\n", (SMversion >= 0x11) ? "capable" : "NOT capable");
+
+        if (SMversion >= 0x11) {
+            gpuInfo[max_gpus++] = devid;
+        }
+    }
+
+    printf("%d GPUs available to run Callback Functions\n", max_gpus);
+
+    heterogeneous_workload *workloads;
+    workloads = (heterogeneous_workload *)malloc(N_workloads * sizeof(heterogeneous_workload));
+    ;
+    thread_barrier = cutCreateBarrier(N_workloads);
+
+    // Main thread spawns a CPU worker thread for each heterogeneous workload
+    printf("Starting %d heterogeneous computing workloads\n", N_workloads);
+
+    for (int i = 0; i < N_workloads; ++i) {
+        workloads[i].id           = i;
+        workloads[i].cudaDeviceID = gpuInfo[i % max_gpus]; // i % N_gpus;
+
+        cutStartThread(launch, &workloads[i]);
+    }
+
+    // Sleep until all workloads have finished
+    cutWaitForBarrier(&thread_barrier);
+    printf("Total of %d workloads finished:\n", N_workloads);
+
+    bool success = true;
+
+    for (int i = 0; i < N_workloads; ++i) {
+        success &= workloads[i].success;
+    }
+
+    printf("%s\n", success ? "Success" : "Failure");
+
+    free(workloads);
+
+    exit(success ? EXIT_SUCCESS : EXIT_FAILURE);
+}