NASP/cuda-samples
84
0
Fork 0
mirror of https://github.com/NVIDIA/cuda-samples.git synced 2026-10-13 00:08:26 +08:00
Code Issues Packages Projects Releases Wiki Activity

CUDA 13.2 samples update (#432)

Browse Source 
- Added Python samples for CUDA Python 1.0 release
- Renamed top-level `Samples` directory to `cpp` to accommodate Python samples.
This commit is contained in:
Dheemanth
2026-05-13 15:13:18 -07:00
committed by GitHub
parent a4526d5229
commit aeab82ff30
2072 changed files with 15077 additions and 1970 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
cpp/2_Concepts_and_Techniques/threadFenceReduction/.vscode/c_cpp_properties.json vendored Normal file
View File

@@ -0,0 +1,18 @@
{
"configurations": [
{
"name": "Linux",
"includePath": [
"${workspaceFolder}/**",
"${workspaceFolder}/../../../Common"
],
"defines": [],
"compilerPath": "/usr/local/cuda/bin/nvcc",
"cStandard": "gnu17",
"cppStandard": "gnu++14",
"intelliSenseMode": "linux-gcc-x64",
"configurationProvider": "ms-vscode.makefile-tools"
}
],
"version": 4
}

7
cpp/2_Concepts_and_Techniques/threadFenceReduction/.vscode/extensions.json vendored Normal file
View File

@@ -0,0 +1,7 @@
{
"recommendations": [
"nvidia.nsight-vscode-edition",
"ms-vscode.cpptools",
"ms-vscode.makefile-tools"
]
}

34
cpp/2_Concepts_and_Techniques/threadFenceReduction/CMakeLists.txt Normal file
View File

@@ -0,0 +1,34 @@
cmake_minimum_required(VERSION 3.20)
list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/../../../cmake/Modules")
project(threadFenceReduction LANGUAGES C CXX CUDA)
find_package(CUDAToolkit REQUIRED)
set(CMAKE_POSITION_INDEPENDENT_CODE ON)
set(CMAKE_CUDA_ARCHITECTURES 75 80 86 87 89 90 100 110 120)
set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Wno-deprecated-gpu-targets")
if(ENABLE_CUDA_DEBUG)
set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -G") # enable cuda-gdb (may significantly affect performance on some targets)
else()
set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -lineinfo") # add line information to all builds for debug tools (exclusive to -G option)
endif()
# Include directories and libraries
include_directories(../../../Common)
# Source file
# Add target for threadFenceReduction
add_executable(threadFenceReduction threadFenceReduction.cu)
target_compile_options(threadFenceReduction PRIVATE $<$<COMPILE_LANGUAGE:CUDA>:--extended-lambda>)
target_compile_features(threadFenceReduction PRIVATE cxx_std_17 cuda_std_17)
set_target_properties(threadFenceReduction PROPERTIES CUDA_SEPARABLE_COMPILATION ON)
# Include installation configuration
include(${CMAKE_CURRENT_SOURCE_DIR}/../../../cmake/InstallSamples.cmake)
setup_samples_install()

32
cpp/2_Concepts_and_Techniques/threadFenceReduction/README.md Normal file
View File

@@ -0,0 +1,32 @@
# threadFenceReduction - threadFenceReduction
## Description
This sample shows how to perform a reduction operation on an array of values using the thread Fence intrinsic to produce a single value in a single kernel (as opposed to two or more kernel calls as shown in the "reduction" CUDA Sample). Single-pass reduction requires global atomic instructions (Compute Capability 2.0 or later) and the _threadfence() intrinsic (CUDA 2.2 or later).
## Key Concepts
Cooperative Groups, Data-Parallel Algorithms, Performance Strategies
## Supported SM Architectures
[SM 5.0 ](https://developer.nvidia.com/cuda-gpus) [SM 5.2 ](https://developer.nvidia.com/cuda-gpus) [SM 5.3 ](https://developer.nvidia.com/cuda-gpus) [SM 6.0 ](https://developer.nvidia.com/cuda-gpus) [SM 6.1 ](https://developer.nvidia.com/cuda-gpus) [SM 7.0 ](https://developer.nvidia.com/cuda-gpus) [SM 7.2 ](https://developer.nvidia.com/cuda-gpus) [SM 7.5 ](https://developer.nvidia.com/cuda-gpus) [SM 8.0 ](https://developer.nvidia.com/cuda-gpus) [SM 8.6 ](https://developer.nvidia.com/cuda-gpus) [SM 8.7 ](https://developer.nvidia.com/cuda-gpus) [SM 8.9 ](https://developer.nvidia.com/cuda-gpus) [SM 9.0 ](https://developer.nvidia.com/cuda-gpus)
## Supported OSes
Linux, Windows
## Supported CPU Architecture
x86_64, armv7l
## CUDA APIs involved
### [CUDA Runtime API](http://docs.nvidia.com/cuda/cuda-runtime-api/index.html)
cudaMemcpy, cudaFree, cudaDeviceSynchronize, cudaMalloc, cudaGetDeviceProperties
## Prerequisites
Download and install the [CUDA Toolkit](https://developer.nvidia.com/cuda-downloads) for your corresponding platform.
## References (for more details)

493
cpp/2_Concepts_and_Techniques/threadFenceReduction/threadFenceReduction.cu Normal file
View File

@@ -0,0 +1,493 @@
/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of NVIDIA CORPORATION nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
* OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
/*
Parallel reduction
This sample shows how to perform a reduction operation on an array of values
to produce a single value in a single kernel (as opposed to two or more
kernel calls as shown in the "reduction" CUDA Sample). Single-pass
reduction requires global atomic instructions (Compute Capability 1.1 or
later) and the __threadfence() intrinsic (CUDA 2.2 or later).
Reductions are a very common computation in parallel algorithms. Any time
an array of values needs to be reduced to a single value using a binary
associative operator, a reduction can be used. Example applications include
statistics computations such as mean and standard deviation, and image
processing applications such as finding the total luminance of an
image.
This code performs sum reductions, but any associative operator such as
min() or max() could also be used.
It assumes the input size is a power of 2.
COMMAND LINE ARGUMENTS
"--shmoo": Test performance for 1 to 32M elements with each of the
7 different kernels
"--n=<N>": Specify the number of elements to reduce (default 1048576)
"--threads=<N>": Specify the number of threads per block (default 128)
"--maxblocks=<N>": Specify the maximum number of thread blocks to launch
(kernel 6 only, default 64)
"--cpufinal": Read back the per-block results and do final sum of block
sums on CPU (default false)
"--cputhresh=<N>": The threshold of number of blocks sums below which to
perform a CPU final reduction (default 1)
"--multipass": Use a multipass reduction instead of a single-pass reduction
*/
// includes, system
#include <math.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
// includes, project
#include <helper_cuda.h>
#include <helper_functions.h>
#define VERSION_MAJOR (CUDART_VERSION / 1000)
#define VERSION_MINOR (CUDART_VERSION % 100) / 10
const char *sSDKsample = "threadFenceReduction";
#if CUDART_VERSION >= 2020
#include "threadFenceReduction_kernel.cuh"
#else
#pragma comment(user, "CUDA 2.2 is required to build for threadFenceReduction")
#endif
////////////////////////////////////////////////////////////////////////////////
// declaration, forward
bool runTest(int argc, char **argv);
extern "C"
{
void reduce(int size, int threads, int blocks, float *d_idata, float *d_odata);
void reduceSinglePass(int size, int threads, int blocks, float *d_idata, float *d_odata);
}
#if CUDART_VERSION < 2020
void reduce(int size, int threads, int blocks, float *d_idata, float *d_odata)
{
printf("reduce(), compiler not supported, aborting tests\n");
}
void reduceSinglePass(int size, int threads, int blocks, float *d_idata, float *d_odata)
{
printf("reduceSinglePass(), compiler not supported, aborting tests\n");
}
#endif
////////////////////////////////////////////////////////////////////////////////
// Program main
////////////////////////////////////////////////////////////////////////////////
int main(int argc, char **argv)
{
cudaDeviceProp deviceProp;
deviceProp.major = 0;
deviceProp.minor = 0;
int dev;
printf("%s Starting...\n\n", sSDKsample);
dev = findCudaDevice(argc, (const char **)argv);
checkCudaErrors(cudaGetDeviceProperties(&deviceProp, dev));
printf("GPU Device supports SM %d.%d compute capability\n\n", deviceProp.major, deviceProp.minor);
bool bTestResult = false;
#if CUDART_VERSION >= 2020
bTestResult = runTest(argc, argv);
#else
print_NVCC_min_spec(sSDKsample, "2.2", "Version 185");
exit(EXIT_SUCCESS);
#endif
exit(bTestResult ? EXIT_SUCCESS : EXIT_FAILURE);
}
////////////////////////////////////////////////////////////////////////////////
//! Compute sum reduction on CPU
//! We use Kahan summation for an accurate sum of large arrays.
//! http://en.wikipedia.org/wiki/Kahan_summation_algorithm
//!
//! @param data pointer to input data
//! @param size number of input data elements
////////////////////////////////////////////////////////////////////////////////
template <class T> T reduceCPU(T *data, int size)
{
T sum = data[0];
T c = (T)0.0;
for (int i = 1; i < size; i++) {
T y = data[i] - c;
T t = sum + y;
c = (t - sum) - y;
sum = t;
}
return sum;
}
unsigned int nextPow2(unsigned int x)
{
--x;
x |= x >> 1;
x |= x >> 2;
x |= x >> 4;
x |= x >> 8;
x |= x >> 16;
return ++x;
}
////////////////////////////////////////////////////////////////////////////////
// Compute the number of threads and blocks to use for the reduction
// We set threads / block to the minimum of maxThreads and n/2.
////////////////////////////////////////////////////////////////////////////////
void getNumBlocksAndThreads(int n, int maxBlocks, int maxThreads, int &blocks, int &threads)
{
if (n == 1) {
threads = 1;
blocks = 1;
}
else {
threads = (n < maxThreads * 2) ? nextPow2(n / 2) : maxThreads;
blocks = max(1, n / (threads * 2));
}
blocks = min(maxBlocks, blocks);
}
////////////////////////////////////////////////////////////////////////////////
// This function performs a reduction of the input data multiple times and
// measures the average reduction time.
////////////////////////////////////////////////////////////////////////////////
float benchmarkReduce(int n,
int numThreads,
int numBlocks,
int maxThreads,
int maxBlocks,
int testIterations,
bool multiPass,
bool cpuFinalReduction,
int cpuFinalThreshold,
StopWatchInterface *timer,
float *h_odata,
float *d_idata,
float *d_odata)
{
float gpu_result = 0;
bool bNeedReadback = true;
cudaError_t error;
for (int i = 0; i < testIterations; ++i) {
gpu_result = 0;
unsigned int retCnt = 0;
error = setRetirementCount(retCnt);
checkCudaErrors(error);
cudaDeviceSynchronize();
sdkStartTimer(&timer);
if (multiPass) {
// execute the kernel
reduce(n, numThreads, numBlocks, d_idata, d_odata);
// check if kernel execution generated an error
getLastCudaError("Kernel execution failed");
if (cpuFinalReduction) {
// sum partial sums from each block on CPU
// copy result from device to host
error = cudaMemcpy(h_odata, d_odata, numBlocks * sizeof(float), cudaMemcpyDeviceToHost);
checkCudaErrors(error);
for (int i = 0; i < numBlocks; i++) {
gpu_result += h_odata[i];
}
bNeedReadback = false;
}
else {
// sum partial block sums on GPU
int s = numBlocks;
while (s > cpuFinalThreshold) {
int threads = 0, blocks = 0;
getNumBlocksAndThreads(s, maxBlocks, maxThreads, blocks, threads);
reduce(s, threads, blocks, d_odata, d_odata);
s = s / (threads * 2);
}
if (s > 1) {
// copy result from device to host
error = cudaMemcpy(h_odata, d_odata, s * sizeof(float), cudaMemcpyDeviceToHost);
checkCudaErrors(error);
for (int i = 0; i < s; i++) {
gpu_result += h_odata[i];
}
bNeedReadback = false;
}
}
}
else {
getLastCudaError("Kernel execution failed");
// execute the kernel
reduceSinglePass(n, numThreads, numBlocks, d_idata, d_odata);
// check if kernel execution generated an error
getLastCudaError("Kernel execution failed");
}
cudaDeviceSynchronize();
sdkStopTimer(&timer);
}
if (bNeedReadback) {
// copy final sum from device to host
error = cudaMemcpy(&gpu_result, d_odata, sizeof(float), cudaMemcpyDeviceToHost);
checkCudaErrors(error);
}
return gpu_result;
}
////////////////////////////////////////////////////////////////////////////////
// This function calls benchmarkReduce multiple times for a range of array sizes
// and prints a report in CSV (comma-separated value) format that can be used
// for generating a "shmoo" plot showing the performance for each kernel
// variation over a wide range of input sizes.
////////////////////////////////////////////////////////////////////////////////
void shmoo(int minN, int maxN, int maxThreads, int maxBlocks)
{
// create random input data on CPU
unsigned int bytes = maxN * sizeof(float);
float *h_idata = (float *)malloc(bytes);
for (int i = 0; i < maxN; i++) {
// Keep the numbers small so we don't get truncation error in the sum
h_idata[i] = (rand() & 0xFF) / (float)RAND_MAX;
}
int maxNumBlocks = min(65535, maxN / maxThreads);
// allocate mem for the result on host side
float *h_odata = (float *)malloc(maxNumBlocks * sizeof(float));
// allocate device memory and data
float *d_idata = NULL;
float *d_odata = NULL;
checkCudaErrors(cudaMalloc((void **)&d_idata, bytes));
checkCudaErrors(cudaMalloc((void **)&d_odata, maxNumBlocks * sizeof(float)));
// copy data directly to device memory
checkCudaErrors(cudaMemcpy(d_idata, h_idata, bytes, cudaMemcpyHostToDevice));
checkCudaErrors(cudaMemcpy(d_odata, h_idata, maxNumBlocks * sizeof(float), cudaMemcpyHostToDevice));
// warm-up
reduce(maxN, maxThreads, maxNumBlocks, d_idata, d_odata);
int testIterations = 100;
StopWatchInterface *timer = NULL;
sdkCreateTimer(&timer);
// print headers
printf("N, %d blocks one pass, %d blocks multipass\n", maxBlocks, maxBlocks);
for (int i = minN; i <= maxN; i *= 2) {
printf("%d, ", i);
for (int multiPass = 0; multiPass <= 1; multiPass++) {
sdkResetTimer(&timer);
int numBlocks = 0;
int numThreads = 0;
getNumBlocksAndThreads(i, maxBlocks, maxThreads, numBlocks, numThreads);
benchmarkReduce(i,
numThreads,
numBlocks,
maxThreads,
maxBlocks,
testIterations,
multiPass == 1,
false,
1,
timer,
h_odata,
d_idata,
d_odata);
float reduceTime = sdkGetAverageTimerValue(&timer);
printf("%f%s", reduceTime, multiPass == 0 ? ", " : "\n");
}
}
printf("\n");
// cleanup
sdkDeleteTimer(&timer);
free(h_idata);
free(h_odata);
cudaFree(d_idata);
cudaFree(d_odata);
}
////////////////////////////////////////////////////////////////////////////////
// The main function which runs the reduction test.
////////////////////////////////////////////////////////////////////////////////
bool runTest(int argc, char **argv)
{
int size = 1 << 20; // number of elements to reduce
int maxThreads = 128; // number of threads per block
int maxBlocks = 64;
bool cpuFinalReduction = false;
int cpuFinalThreshold = 1;
bool multipass = false;
bool bTestResult = false;
if (checkCmdLineFlag(argc, (const char **)argv, "n")) {
size = getCmdLineArgumentInt(argc, (const char **)argv, "n");
}
if (checkCmdLineFlag(argc, (const char **)argv, "threads")) {
maxThreads = getCmdLineArgumentInt(argc, (const char **)argv, "threads");
}
if (checkCmdLineFlag(argc, (const char **)argv, "maxblocks")) {
maxBlocks = getCmdLineArgumentInt(argc, (const char **)argv, "maxblocks");
}
printf("%d elements\n", size);
printf("%d threads (max)\n", maxThreads);
cpuFinalReduction = checkCmdLineFlag(argc, (const char **)argv, "cpufinal");
multipass = checkCmdLineFlag(argc, (const char **)argv, "multipass");
if (checkCmdLineFlag(argc, (const char **)argv, "cputhresh")) {
cpuFinalThreshold = getCmdLineArgumentInt(argc, (const char **)argv, "cputhresh");
}
bool runShmoo = checkCmdLineFlag(argc, (const char **)argv, "shmoo");
if (runShmoo) {
shmoo(1, 33554432, maxThreads, maxBlocks);
}
else {
// create random input data on CPU
unsigned int bytes = size * sizeof(float);
float *h_idata = (float *)malloc(bytes);
for (int i = 0; i < size; i++) {
// Keep the numbers small so we don't get truncation error in the sum
h_idata[i] = (rand() & 0xFF) / (float)RAND_MAX;
}
int numBlocks = 0;
int numThreads = 0;
getNumBlocksAndThreads(size, maxBlocks, maxThreads, numBlocks, numThreads);
if (numBlocks == 1) {
cpuFinalThreshold = 1;
}
// allocate mem for the result on host side
float *h_odata = (float *)malloc(numBlocks * sizeof(float));
printf("%d blocks\n", numBlocks);
// allocate device memory and data
float *d_idata = NULL;
float *d_odata = NULL;
checkCudaErrors(cudaMalloc((void **)&d_idata, bytes));
checkCudaErrors(cudaMalloc((void **)&d_odata, numBlocks * sizeof(float)));
// copy data directly to device memory
checkCudaErrors(cudaMemcpy(d_idata, h_idata, bytes, cudaMemcpyHostToDevice));
checkCudaErrors(cudaMemcpy(d_odata, h_idata, numBlocks * sizeof(float), cudaMemcpyHostToDevice));
// warm-up
reduce(size, numThreads, numBlocks, d_idata, d_odata);
int testIterations = 100;
StopWatchInterface *timer = 0;
sdkCreateTimer(&timer);
float gpu_result = 0;
gpu_result = benchmarkReduce(size,
numThreads,
numBlocks,
maxThreads,
maxBlocks,
testIterations,
multipass,
cpuFinalReduction,
cpuFinalThreshold,
timer,
h_odata,
d_idata,
d_odata);
float reduceTime = sdkGetAverageTimerValue(&timer);
printf("Average time: %f ms\n", reduceTime);
printf("Bandwidth: %f GB/s\n\n", (size * sizeof(int)) / (reduceTime * 1.0e6));
// compute reference solution
float cpu_result = reduceCPU<float>(h_idata, size);
printf("GPU result = %0.12f\n", gpu_result);
printf("CPU result = %0.12f\n", cpu_result);
double threshold = 1e-8 * size;
double diff = abs((double)gpu_result - (double)cpu_result);
bTestResult = (diff < threshold);
// cleanup
sdkDeleteTimer(&timer);
free(h_idata);
free(h_odata);
cudaFree(d_idata);
cudaFree(d_odata);
}
return bTestResult;
}

33
cpp/2_Concepts_and_Techniques/threadFenceReduction/threadFenceReduction.h Normal file
View File

@@ -0,0 +1,33 @@
/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of NVIDIA CORPORATION nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
* OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef __REDUCTION_H__
#define __REDUCTION_H__
cudaError_t setRetirementCount(int retCnt);
#endif

403
cpp/2_Concepts_and_Techniques/threadFenceReduction/threadFenceReduction_kernel.cuh Normal file
View File

@@ -0,0 +1,403 @@
/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of NVIDIA CORPORATION nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
* OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
/*
Parallel reduction kernels
*/
#ifndef _REDUCE_KERNEL_H_
#define _REDUCE_KERNEL_H_
#include <cooperative_groups.h>
#include <cuda_runtime_api.h>
namespace cg = cooperative_groups;
/*
Parallel sum reduction using shared memory
- takes log(n) steps for n input elements
- uses n/2 threads
- only works for power-of-2 arrays
This version adds multiple elements per thread sequentially. This reduces
the overall cost of the algorithm while keeping the work complexity O(n) and
the step complexity O(log n). (Brent's Theorem optimization)
See the CUDA SDK "reduction" sample for more information.
*/
template <unsigned int blockSize>
__device__ void reduceBlock(volatile float *sdata, float mySum, const unsigned int tid, cg::thread_block cta)
{
cg::thread_block_tile<32> tile32 = cg::tiled_partition<32>(cta);
sdata[tid] = mySum;
cg::sync(tile32);
const int VEC = 32;
const int vid = tid & (VEC - 1);
float beta = mySum;
float temp;
for (int i = VEC / 2; i > 0; i >>= 1) {
if (vid < i) {
temp = sdata[tid + i];
beta += temp;
sdata[tid] = beta;
}
cg::sync(tile32);
}
cg::sync(cta);
if (cta.thread_rank() == 0) {
beta = 0;
for (int i = 0; i < blockDim.x; i += VEC) {
beta += sdata[i];
}
sdata[0] = beta;
}
cg::sync(cta);
}
template <unsigned int blockSize, bool nIsPow2>
__device__ void reduceBlocks(const float *g_idata, float *g_odata, unsigned int n, cg::thread_block cta)
{
extern __shared__ float sdata[];
// perform first level of reduction,
// reading from global memory, writing to shared memory
unsigned int tid = threadIdx.x;
unsigned int i = blockIdx.x * (blockSize * 2) + threadIdx.x;
unsigned int gridSize = blockSize * 2 * gridDim.x;
float mySum = 0;
// we reduce multiple elements per thread. The number is determined by the
// number of active thread blocks (via gridDim). More blocks will result
// in a larger gridSize and therefore fewer elements per thread
while (i < n) {
mySum += g_idata[i];
// ensure we don't read out of bounds -- this is optimized away for powerOf2
// sized arrays
if (nIsPow2 || i + blockSize < n)
mySum += g_idata[i + blockSize];
i += gridSize;
}
// do reduction in shared mem
reduceBlock<blockSize>(sdata, mySum, tid, cta);
// write result for this block to global mem
if (tid == 0)
g_odata[blockIdx.x] = sdata[0];
}
template <unsigned int blockSize, bool nIsPow2>
__global__ void reduceMultiPass(const float *g_idata, float *g_odata, unsigned int n)
{
// Handle to thread block group
cg::thread_block cta = cg::this_thread_block();
reduceBlocks<blockSize, nIsPow2>(g_idata, g_odata, n, cta);
}
// Global variable used by reduceSinglePass to count how many blocks have
// finished
__device__ unsigned int retirementCount = 0;
cudaError_t setRetirementCount(int retCnt)
{
return cudaMemcpyToSymbol(retirementCount, &retCnt, sizeof(unsigned int), 0, cudaMemcpyHostToDevice);
}
// This reduction kernel reduces an arbitrary size array in a single kernel
// invocation It does so by keeping track of how many blocks have finished.
// After each thread block completes the reduction of its own block of data, it
// "takes a ticket" by atomically incrementing a global counter. If the ticket
// value is equal to the number of thread blocks, then the block holding the
// ticket knows that it is the last block to finish. This last block is
// responsible for summing the results of all the other blocks.
//
// In order for this to work, we must be sure that before a block takes a
// ticket, all of its memory transactions have completed. This is what
// __threadfence() does -- it blocks until the results of all outstanding memory
// transactions within the calling thread are visible to all other threads.
//
// For more details on the reduction algorithm (notably the multi-pass
// approach), see the "reduction" sample in the CUDA SDK.
template <unsigned int blockSize, bool nIsPow2>
__global__ void reduceSinglePass(const float *g_idata, float *g_odata, unsigned int n)
{
// Handle to thread block group
cg::thread_block cta = cg::this_thread_block();
//
// PHASE 1: Process all inputs assigned to this block
//
reduceBlocks<blockSize, nIsPow2>(g_idata, g_odata, n, cta);
//
// PHASE 2: Last block finished will process all partial sums
//
if (gridDim.x > 1) {
const unsigned int tid = threadIdx.x;
__shared__ bool amLast;
extern float __shared__ smem[];
// wait until all outstanding memory instructions in this thread are
// finished
__threadfence();
// Thread 0 takes a ticket
if (tid == 0) {
unsigned int ticket = atomicInc(&retirementCount, gridDim.x);
// If the ticket ID is equal to the number of blocks, we are the last
// block!
amLast = (ticket == gridDim.x - 1);
}
cg::sync(cta);
// The last block sums the results of all other blocks
if (amLast) {
int i = tid;
float mySum = 0;
while (i < gridDim.x) {
mySum += g_odata[i];
i += blockSize;
}
reduceBlock<blockSize>(smem, mySum, tid, cta);
if (tid == 0) {
g_odata[0] = smem[0];
// reset retirement count so that next run succeeds
retirementCount = 0;
}
}
}
}
bool isPow2(unsigned int x) { return ((x & (x - 1)) == 0); }
////////////////////////////////////////////////////////////////////////////////
// Wrapper function for kernel launch
////////////////////////////////////////////////////////////////////////////////
extern "C" void reduce(int size, int threads, int blocks, float *d_idata, float *d_odata)
{
dim3 dimBlock(threads, 1, 1);
dim3 dimGrid(blocks, 1, 1);
int smemSize = (threads <= 32) ? 2 * threads * sizeof(float) : threads * sizeof(float);
// choose which of the optimized versions of reduction to launch
if (isPow2(size)) {
switch (threads) {
case 512:
reduceMultiPass<512, true><<<dimGrid, dimBlock, smemSize>>>(d_idata, d_odata, size);
break;
case 256:
reduceMultiPass<256, true><<<dimGrid, dimBlock, smemSize>>>(d_idata, d_odata, size);
break;
case 128:
reduceMultiPass<128, true><<<dimGrid, dimBlock, smemSize>>>(d_idata, d_odata, size);
break;
case 64:
reduceMultiPass<64, true><<<dimGrid, dimBlock, smemSize>>>(d_idata, d_odata, size);
break;
case 32:
reduceMultiPass<32, true><<<dimGrid, dimBlock, smemSize>>>(d_idata, d_odata, size);
break;
case 16:
reduceMultiPass<16, true><<<dimGrid, dimBlock, smemSize>>>(d_idata, d_odata, size);
break;
case 8:
reduceMultiPass<8, true><<<dimGrid, dimBlock, smemSize>>>(d_idata, d_odata, size);
break;
case 4:
reduceMultiPass<4, true><<<dimGrid, dimBlock, smemSize>>>(d_idata, d_odata, size);
break;
case 2:
reduceMultiPass<2, true><<<dimGrid, dimBlock, smemSize>>>(d_idata, d_odata, size);
break;
case 1:
reduceMultiPass<1, true><<<dimGrid, dimBlock, smemSize>>>(d_idata, d_odata, size);
break;
}
}
else {
switch (threads) {
case 512:
reduceMultiPass<512, false><<<dimGrid, dimBlock, smemSize>>>(d_idata, d_odata, size);
break;
case 256:
reduceMultiPass<256, false><<<dimGrid, dimBlock, smemSize>>>(d_idata, d_odata, size);
break;
case 128:
reduceMultiPass<128, false><<<dimGrid, dimBlock, smemSize>>>(d_idata, d_odata, size);
break;
case 64:
reduceMultiPass<64, false><<<dimGrid, dimBlock, smemSize>>>(d_idata, d_odata, size);
break;
case 32:
reduceMultiPass<32, false><<<dimGrid, dimBlock, smemSize>>>(d_idata, d_odata, size);
break;
case 16:
reduceMultiPass<16, false><<<dimGrid, dimBlock, smemSize>>>(d_idata, d_odata, size);
break;
case 8:
reduceMultiPass<8, false><<<dimGrid, dimBlock, smemSize>>>(d_idata, d_odata, size);
break;
case 4:
reduceMultiPass<4, false><<<dimGrid, dimBlock, smemSize>>>(d_idata, d_odata, size);
break;
case 2:
reduceMultiPass<2, false><<<dimGrid, dimBlock, smemSize>>>(d_idata, d_odata, size);
break;
case 1:
reduceMultiPass<1, false><<<dimGrid, dimBlock, smemSize>>>(d_idata, d_odata, size);
break;
}
}
}
extern "C" void reduceSinglePass(int size, int threads, int blocks, float *d_idata, float *d_odata)
{
dim3 dimBlock(threads, 1, 1);
dim3 dimGrid(blocks, 1, 1);
int smemSize = threads * sizeof(float);
// choose which of the optimized versions of reduction to launch
if (isPow2(size)) {
switch (threads) {
case 512:
reduceSinglePass<512, true><<<dimGrid, dimBlock, smemSize>>>(d_idata, d_odata, size);
break;
case 256:
reduceSinglePass<256, true><<<dimGrid, dimBlock, smemSize>>>(d_idata, d_odata, size);
break;
case 128:
reduceSinglePass<128, true><<<dimGrid, dimBlock, smemSize>>>(d_idata, d_odata, size);
break;
case 64:
reduceSinglePass<64, true><<<dimGrid, dimBlock, smemSize>>>(d_idata, d_odata, size);
break;
case 32:
reduceSinglePass<32, true><<<dimGrid, dimBlock, smemSize>>>(d_idata, d_odata, size);
break;
case 16:
reduceSinglePass<16, true><<<dimGrid, dimBlock, smemSize>>>(d_idata, d_odata, size);
break;
case 8:
reduceSinglePass<8, true><<<dimGrid, dimBlock, smemSize>>>(d_idata, d_odata, size);
break;
case 4:
reduceSinglePass<4, true><<<dimGrid, dimBlock, smemSize>>>(d_idata, d_odata, size);
break;
case 2:
reduceSinglePass<2, true><<<dimGrid, dimBlock, smemSize>>>(d_idata, d_odata, size);
break;
case 1:
reduceSinglePass<1, true><<<dimGrid, dimBlock, smemSize>>>(d_idata, d_odata, size);
break;
}
}
else {
switch (threads) {
case 512:
reduceSinglePass<512, false><<<dimGrid, dimBlock, smemSize>>>(d_idata, d_odata, size);
break;
case 256:
reduceSinglePass<256, false><<<dimGrid, dimBlock, smemSize>>>(d_idata, d_odata, size);
break;
case 128:
reduceSinglePass<128, false><<<dimGrid, dimBlock, smemSize>>>(d_idata, d_odata, size);
break;
case 64:
reduceSinglePass<64, false><<<dimGrid, dimBlock, smemSize>>>(d_idata, d_odata, size);
break;
case 32:
reduceSinglePass<32, false><<<dimGrid, dimBlock, smemSize>>>(d_idata, d_odata, size);
break;
case 16:
reduceSinglePass<16, false><<<dimGrid, dimBlock, smemSize>>>(d_idata, d_odata, size);
break;
case 8:
reduceSinglePass<8, false><<<dimGrid, dimBlock, smemSize>>>(d_idata, d_odata, size);
break;
case 4:
reduceSinglePass<4, false><<<dimGrid, dimBlock, smemSize>>>(d_idata, d_odata, size);
break;
case 2:
reduceSinglePass<2, false><<<dimGrid, dimBlock, smemSize>>>(d_idata, d_odata, size);
break;
case 1:
reduceSinglePass<1, false><<<dimGrid, dimBlock, smemSize>>>(d_idata, d_odata, size);
break;
}
}
}
#endif // #ifndef _REDUCE_KERNEL_H_