NASP/cuda-samples
84
0
Fork 0
mirror of https://github.com/NVIDIA/cuda-samples.git synced 2026-10-13 00:08:26 +08:00
Code Issues Packages Projects Releases Wiki Activity

CUDA 13.2 samples update (#432)

Browse Source 
- Added Python samples for CUDA Python 1.0 release
- Renamed top-level `Samples` directory to `cpp` to accommodate Python samples.
This commit is contained in:
Dheemanth
2026-05-13 15:13:18 -07:00
committed by GitHub
parent a4526d5229
commit aeab82ff30
2072 changed files with 15077 additions and 1970 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
cpp/3_CUDA_Features/cdpAdvancedQuicksort/.vscode/c_cpp_properties.json vendored Normal file
View File

@@ -0,0 +1,18 @@
{
"configurations": [
{
"name": "Linux",
"includePath": [
"${workspaceFolder}/**",
"${workspaceFolder}/../../../Common"
],
"defines": [],
"compilerPath": "/usr/local/cuda/bin/nvcc",
"cStandard": "gnu17",
"cppStandard": "gnu++14",
"intelliSenseMode": "linux-gcc-x64",
"configurationProvider": "ms-vscode.makefile-tools"
}
],
"version": 4
}

7
cpp/3_CUDA_Features/cdpAdvancedQuicksort/.vscode/extensions.json vendored Normal file
View File

@@ -0,0 +1,7 @@
{
"recommendations": [
"nvidia.nsight-vscode-edition",
"ms-vscode.cpptools",
"ms-vscode.makefile-tools"
]
}

44
cpp/3_CUDA_Features/cdpAdvancedQuicksort/CMakeLists.txt Normal file
View File

@@ -0,0 +1,44 @@
cmake_minimum_required(VERSION 3.20)
list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/../../../cmake/Modules")
project(cdpAdvancedQuicksort LANGUAGES C CXX CUDA)
find_package(CUDAToolkit REQUIRED)
set(CMAKE_POSITION_INDEPENDENT_CODE ON)
# The aarch64/sbsa_aarch64 CUDA toolkit are support on Tegra since 13.0, so need to check which version of the toolkit is installed
string(FIND "${CUDAToolkit_INCLUDE_DIRS}" "aarch64-linux" _aarch64_linux_ctk)
string(FIND "${CUDAToolkit_INCLUDE_DIRS}" "aarch64-qnx" _aarch64_qnx_ctk)
if(CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND (NOT _aarch64_linux_ctk EQUAL -1 OR NOT _aarch64_qnx_ctk EQUAL -1))
set(CMAKE_CUDA_ARCHITECTURES 87 110)
else()
set(CMAKE_CUDA_ARCHITECTURES 75 80 86 89 90 100 110 120)
endif()
set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Wno-deprecated-gpu-targets")
if(ENABLE_CUDA_DEBUG)
set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -G") # enable cuda-gdb (may significantly affect performance on some targets)
set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -maxrregcount=64") # Limit register usage to 64 for the 'big_bitonicsort kernel
else()
set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -lineinfo") # add line information to all builds for debug tools (exclusive to -G option)
endif()
# Include directories and libraries
include_directories(../../../Common)
# Source file
# Add target for cdpAdvancedQuicksort
add_executable(cdpAdvancedQuicksort cdpAdvancedQuicksort.cu cdpBitonicSort.cu)
target_compile_options(cdpAdvancedQuicksort PRIVATE $<$<COMPILE_LANGUAGE:CUDA>:--extended-lambda>)
target_compile_features(cdpAdvancedQuicksort PRIVATE cxx_std_17 cuda_std_17)
set_target_properties(cdpAdvancedQuicksort PROPERTIES CUDA_SEPARABLE_COMPILATION ON)
# Include installation configuration
include(${CMAKE_CURRENT_SOURCE_DIR}/../../../cmake/InstallSamples.cmake)
setup_samples_install()

34
cpp/3_CUDA_Features/cdpAdvancedQuicksort/README.md Normal file
View File

@@ -0,0 +1,34 @@
# cdpAdvancedQuicksort - Advanced Quicksort (CUDA Dynamic Parallelism)
## Description
This sample demonstrates an advanced quicksort implemented using CUDA Dynamic Parallelism. This sample requires devices with compute capability 3.5 or higher.
## Key Concepts
Cooperative Groups, CUDA Dynamic Parallelism
## Supported SM Architectures
## Supported OSes
Linux, Windows
## Supported CPU Architecture
x86_64, armv7l
## CUDA APIs involved
### [CUDA Runtime API](http://docs.nvidia.com/cuda/cuda-runtime-api/index.html)
cudaStreamCreateWithFlags, cudaMemcpy, cudaMemcpyAsync, cudaFree, cudaGetErrorString, cudaGetLastError, cudaPeekAtLastError, cudaDeviceSynchronize, cudaEventRecord, cudaMemset, cudaMalloc, cudaEventElapsedTime, cudaGetDeviceProperties, cudaEventCreate
## Dependencies needed to build/run
[CDP](../../../README.md#cuda-dynamic-parallellism)
## Prerequisites
Download and install the [CUDA Toolkit](https://developer.nvidia.com/cuda-downloads) for your corresponding platform.
Make sure the dependencies mentioned in [Dependencies]() section above are installed.
## References (for more details)

584
cpp/3_CUDA_Features/cdpAdvancedQuicksort/cdpAdvancedQuicksort.cu Normal file
View File

@@ -0,0 +1,584 @@
/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of NVIDIA CORPORATION nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
* OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
////////////////////////////////////////////////////////////////////////////////
//
// cdpAdvancedQuicksort.cu
//
// Implementation of a parallel quicksort in CUDA. It comes in
// several parts:
//
// 1. A small-set insertion sort. We do this on any set with <=32 elements
// 2. A partitioning kernel, which - given a pivot - separates an input
// array into elements <=pivot, and >pivot. Two quicksorts will then
// be launched to resolve each of these.
// 3. A quicksort co-ordinator, which figures out what kernels to launch
// and when.
//
////////////////////////////////////////////////////////////////////////////////
#include <cooperative_groups.h>
#include <thrust/device_vector.h>
#include <thrust/random.h>
namespace cg = cooperative_groups;
#include <helper_cuda.h>
#include <helper_string.h>
#include "cdpQuicksort.h"
////////////////////////////////////////////////////////////////////////////////
// Inline PTX call to return index of highest non-zero bit in a word
////////////////////////////////////////////////////////////////////////////////
static __device__ __forceinline__ unsigned int __qsflo(unsigned int word)
{
unsigned int ret;
asm volatile("bfind.u32 %0, %1;" : "=r"(ret) : "r"(word));
return ret;
}
////////////////////////////////////////////////////////////////////////////////
//
// ringbufAlloc
//
// Allocates from a ringbuffer. Allows for not failing when we run out
// of stack for tracking the offset counts for each sort subsection.
//
// We use the atomicMax trick to allow out-of-order retirement. If we
// hit the size limit on the ringbuffer, then we spin-wait for people
// to complete.
//
////////////////////////////////////////////////////////////////////////////////
template <typename T> static __device__ T *ringbufAlloc(qsortRingbuf *ringbuf)
{
// Wait for there to be space in the ring buffer. We'll retry only a fixed
// number of times and then fail, to avoid an out-of-memory deadlock.
unsigned int loop = 10000;
while (((ringbuf->head - ringbuf->tail) >= ringbuf->stacksize) && (loop-- > 0))
;
if (loop == 0)
return NULL;
// Note that the element includes a little index book-keeping, for freeing
// later.
unsigned int index = atomicAdd((unsigned int *)&ringbuf->head, 1);
T *ret = (T *)(ringbuf->stackbase) + (index & (ringbuf->stacksize - 1));
ret->index = index;
return ret;
}
////////////////////////////////////////////////////////////////////////////////
//
// ringBufFree
//
// Releases an element from the ring buffer. If every element is released
// up to and including this one, we can advance the tail to indicate that
// space is now available.
//
////////////////////////////////////////////////////////////////////////////////
template <typename T> static __device__ void ringbufFree(qsortRingbuf *ringbuf, T *data)
{
unsigned int index = data->index; // Non-wrapped index to free
unsigned int count = atomicAdd((unsigned int *)&(ringbuf->count), 1) + 1;
unsigned int max = atomicMax((unsigned int *)&(ringbuf->max), index + 1);
// Update the tail if need be. Note we update "max" to be the new value in
// ringbuf->max
if (max < (index + 1))
max = index + 1;
if (max == count)
atomicMax((unsigned int *)&(ringbuf->tail), count);
}
////////////////////////////////////////////////////////////////////////////////
//
// qsort_warp
//
// Simplest possible implementation, does a per-warp quicksort with no
// inter-warp
// communication. This has a high atomic issue rate, but the rest should
// actually
// be fairly quick because of low work per thread.
//
// A warp finds its section of the data, then writes all data <pivot to one
// buffer and all data >pivot to the other. Atomics are used to get a unique
// section of the buffer.
//
// Obvious optimisation: do multiple chunks per warp, to increase in-flight
// loads
// and cover the instruction overhead.
//
////////////////////////////////////////////////////////////////////////////////
__global__ void qsort_warp(unsigned *indata,
unsigned *outdata,
unsigned int offset,
unsigned int len,
qsortAtomicData *atomicData,
qsortRingbuf *atomicDataStack,
unsigned int source_is_indata,
unsigned int depth)
{
// Handle to thread block group
cg::thread_block cta = cg::this_thread_block();
// Find my data offset, based on warp ID
unsigned int thread_id = threadIdx.x + (blockIdx.x << QSORT_BLOCKSIZE_SHIFT);
// unsigned int warp_id = threadIdx.x >> 5; // Used for debug only
unsigned int lane_id = threadIdx.x & (warpSize - 1);
// Exit if I'm outside the range of sort to be done
if (thread_id >= len)
return;
//
// First part of the algorithm. Each warp counts the number of elements that
// are
// greater/less than the pivot.
//
// When a warp knows its count, it updates an atomic counter.
//
// Read in the data and the pivot. Arbitrary pivot selection for now.
unsigned pivot = indata[offset + len / 2];
unsigned data = indata[offset + thread_id];
// Count how many are <= and how many are > pivot.
// If all are <= pivot then we adjust the comparison
// because otherwise the sort will move nothing and
// we'll iterate forever.
cg::coalesced_group active = cg::coalesced_threads();
unsigned int greater = (data > pivot);
unsigned int gt_mask = active.ballot(greater);
if (gt_mask == 0) {
greater = (data >= pivot);
gt_mask = active.ballot(greater); // Must re-ballot for adjusted comparator
}
unsigned int lt_mask = active.ballot(!greater);
unsigned int gt_count = __popc(gt_mask);
unsigned int lt_count = __popc(lt_mask);
// Atomically adjust the lt_ and gt_offsets by this amount. Only one thread
// need do this. Share the result using shfl
unsigned int lt_offset, gt_offset;
if (lane_id == 0) {
if (lt_count > 0)
lt_offset = atomicAdd((unsigned int *)&atomicData->lt_offset, lt_count);
if (gt_count > 0)
gt_offset = len - (atomicAdd((unsigned int *)&atomicData->gt_offset, gt_count) + gt_count);
}
lt_offset = active.shfl((int)lt_offset, 0); // Everyone pulls the offsets from lane 0
gt_offset = active.shfl((int)gt_offset, 0);
// Now compute my own personal offset within this. I need to know how many
// threads with a lane ID less than mine are going to write to the same buffer
// as me. We can use popc to implement a single-operation warp scan in this
// case.
unsigned lane_mask_lt;
asm("mov.u32 %0, %%lanemask_lt;" : "=r"(lane_mask_lt));
unsigned int my_mask = greater ? gt_mask : lt_mask;
unsigned int my_offset = __popc(my_mask & lane_mask_lt);
// Move data.
my_offset += greater ? gt_offset : lt_offset;
outdata[offset + my_offset] = data;
// Count up if we're the last warp in. If so, then Kepler will launch the next
// set of sorts directly from here.
if (lane_id == 0) {
// Count "elements written". If I wrote the last one, then trigger the next
// qsorts
unsigned int mycount = lt_count + gt_count;
if (atomicAdd((unsigned int *)&atomicData->sorted_count, mycount) + mycount == len) {
// We're the last warp to do any sorting. Therefore it's up to us to
// launch the next stage.
unsigned int lt_len = atomicData->lt_offset;
unsigned int gt_len = atomicData->gt_offset;
cudaStream_t lstream, rstream;
cudaStreamCreateWithFlags(&lstream, cudaStreamNonBlocking);
cudaStreamCreateWithFlags(&rstream, cudaStreamNonBlocking);
// Begin by freeing our atomicData storage. It's better for the ringbuffer
// algorithm
// if we free when we're done, rather than re-using (makes for less
// fragmentation).
ringbufFree<qsortAtomicData>(atomicDataStack, atomicData);
// Exceptional case: if "lt_len" is zero, then all values in the batch
// are equal. We are then done (may need to copy into correct buffer,
// though)
if (lt_len == 0) {
if (source_is_indata)
cudaMemcpyAsync(indata + offset,
outdata + offset,
gt_len * sizeof(unsigned),
cudaMemcpyDeviceToDevice,
lstream);
return;
}
// Start with lower half first
if (lt_len > BITONICSORT_LEN) {
// If we've exceeded maximum depth, fall through to backup
// big_bitonicsort
if (depth >= QSORT_MAXDEPTH) {
// The final bitonic stage sorts in-place in "outdata". We therefore
// re-use "indata" as the out-of-range tracking buffer. For (2^n)+1
// elements we need (2^(n+1)) bytes of oor buffer. The backup qsort
// buffer is at least this large when sizeof(QTYPE) >= 2.
big_bitonicsort<<<1, BITONICSORT_LEN, 0, lstream>>>(
outdata, source_is_indata ? indata : outdata, indata, offset, lt_len);
}
else {
// Launch another quicksort. We need to allocate more storage for the
// atomic data.
if ((atomicData = ringbufAlloc<qsortAtomicData>(atomicDataStack)) == NULL)
printf("Stack-allocation error. Failing left child launch.\n");
else {
atomicData->lt_offset = atomicData->gt_offset = atomicData->sorted_count = 0;
unsigned int numblocks =
(unsigned int)(lt_len + (QSORT_BLOCKSIZE - 1)) >> QSORT_BLOCKSIZE_SHIFT;
qsort_warp<<<numblocks, QSORT_BLOCKSIZE, 0, lstream>>>(
outdata, indata, offset, lt_len, atomicData, atomicDataStack, !source_is_indata, depth + 1);
}
}
}
else if (lt_len > 1) {
// Final stage uses a bitonic sort instead. It's important to
// make sure the final stage ends up in the correct (original) buffer.
// We launch the smallest power-of-2 number of threads that we can.
unsigned int bitonic_len = 1 << (__qsflo(lt_len - 1U) + 1);
bitonicsort<<<1, bitonic_len, 0, lstream>>>(
outdata, source_is_indata ? indata : outdata, offset, lt_len);
}
// Finally, if we sorted just one single element, we must still make
// sure that it winds up in the correct place.
else if (source_is_indata && (lt_len == 1))
indata[offset] = outdata[offset];
if (cudaPeekAtLastError() != cudaSuccess)
printf("Left-side launch fail: %s\n", cudaGetErrorString(cudaGetLastError()));
// Now the upper half.
if (gt_len > BITONICSORT_LEN) {
// If we've exceeded maximum depth, fall through to backup
// big_bitonicsort
if (depth >= QSORT_MAXDEPTH)
big_bitonicsort<<<1, BITONICSORT_LEN, 0, rstream>>>(
outdata, source_is_indata ? indata : outdata, indata, offset + lt_len, gt_len);
else {
// Allocate new atomic storage for this launch
if ((atomicData = ringbufAlloc<qsortAtomicData>(atomicDataStack)) == NULL)
printf("Stack allocation error! Failing right-side launch.\n");
else {
atomicData->lt_offset = atomicData->gt_offset = atomicData->sorted_count = 0;
unsigned int numblocks =
(unsigned int)(gt_len + (QSORT_BLOCKSIZE - 1)) >> QSORT_BLOCKSIZE_SHIFT;
qsort_warp<<<numblocks, QSORT_BLOCKSIZE, 0, rstream>>>(outdata,
indata,
offset + lt_len,
gt_len,
atomicData,
atomicDataStack,
!source_is_indata,
depth + 1);
}
}
}
else if (gt_len > 1) {
unsigned int bitonic_len = 1 << (__qsflo(gt_len - 1U) + 1);
bitonicsort<<<1, bitonic_len, 0, rstream>>>(
outdata, source_is_indata ? indata : outdata, offset + lt_len, gt_len);
}
else if (source_is_indata && (gt_len == 1))
indata[offset + lt_len] = outdata[offset + lt_len];
if (cudaPeekAtLastError() != cudaSuccess)
printf("Right-side launch fail: %s\n", cudaGetErrorString(cudaGetLastError()));
}
}
}
////////////////////////////////////////////////////////////////////////////////
//
// run_quicksort
//
// Host-side code to run the Kepler version of quicksort. It's pretty
// simple, because all launch control is handled on the device via CDP.
//
// All parallel quicksorts require an equal-sized scratch buffer. This
// must be passed in ahead of time.
//
// Returns the time elapsed for the sort.
//
////////////////////////////////////////////////////////////////////////////////
float run_quicksort_cdp(unsigned *gpudata, unsigned *scratchdata, unsigned int count, cudaStream_t stream)
{
unsigned int stacksize = QSORT_STACK_ELEMS;
// This is the stack, for atomic tracking of each sort's status
qsortAtomicData *gpustack;
checkCudaErrors(cudaMalloc((void **)&gpustack, stacksize * sizeof(qsortAtomicData)));
checkCudaErrors(cudaMemset(gpustack, 0, sizeof(qsortAtomicData))); // Only need set first entry to 0
// Create the memory ringbuffer used for handling the stack.
// Initialise everything to where it needs to be.
qsortRingbuf buf;
qsortRingbuf *ringbuf;
checkCudaErrors(cudaMalloc((void **)&ringbuf, sizeof(qsortRingbuf)));
buf.head = 1; // We start with one allocation
buf.tail = 0;
buf.count = 0;
buf.max = 0;
buf.stacksize = stacksize;
buf.stackbase = gpustack;
checkCudaErrors(cudaMemcpy(ringbuf, &buf, sizeof(buf), cudaMemcpyHostToDevice));
// Timing events...
cudaEvent_t ev1, ev2;
checkCudaErrors(cudaEventCreate(&ev1));
checkCudaErrors(cudaEventCreate(&ev2));
checkCudaErrors(cudaEventRecord(ev1));
// Now we trivially launch the qsort kernel
if (count > BITONICSORT_LEN) {
unsigned int numblocks = (unsigned int)(count + (QSORT_BLOCKSIZE - 1)) >> QSORT_BLOCKSIZE_SHIFT;
qsort_warp<<<numblocks, QSORT_BLOCKSIZE, 0, stream>>>(
gpudata, scratchdata, 0U, count, gpustack, ringbuf, true, 0);
}
else {
bitonicsort<<<1, BITONICSORT_LEN>>>(gpudata, gpudata, 0, count);
}
checkCudaErrors(cudaGetLastError());
checkCudaErrors(cudaEventRecord(ev2));
checkCudaErrors(cudaDeviceSynchronize());
float elapse = 0.0f;
if (cudaPeekAtLastError() != cudaSuccess)
printf("Launch failure: %s\n", cudaGetErrorString(cudaGetLastError()));
else
checkCudaErrors(cudaEventElapsedTime(&elapse, ev1, ev2));
// Sanity check that the stack allocator is doing the right thing
checkCudaErrors(cudaMemcpy(&buf, ringbuf, sizeof(*ringbuf), cudaMemcpyDeviceToHost));
if (count > BITONICSORT_LEN && buf.head != buf.tail) {
printf("Stack allocation error!\nRingbuf:\n");
printf("\t head = %u\n", buf.head);
printf("\t tail = %u\n", buf.tail);
printf("\tcount = %u\n", buf.count);
printf("\t max = %u\n", buf.max);
}
// Release our stack data once we're done
checkCudaErrors(cudaFree(ringbuf));
checkCudaErrors(cudaFree(gpustack));
return elapse;
}
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
int run_qsort(unsigned int size, int seed, int debug, int loop, int verbose)
{
if (seed > 0)
srand(seed);
// Create and set up our test
unsigned *gpudata, *scratchdata;
checkCudaErrors(cudaMalloc((void **)&gpudata, size * sizeof(unsigned)));
checkCudaErrors(cudaMalloc((void **)&scratchdata, size * sizeof(unsigned)));
// Create CPU data.
unsigned *data = new unsigned[size];
unsigned int min = loop ? loop : size;
unsigned int max = size;
loop = (loop == 0) ? 1 : loop;
for (size = min; size <= max; size += loop) {
if (verbose)
printf(" Input: ");
for (unsigned int i = 0; i < size; i++) {
// Build data 8 bits at a time
data[i] = 0;
char *ptr = (char *)&(data[i]);
for (unsigned j = 0; j < sizeof(unsigned); j++) {
// Easy-to-read data in debug mode
if (debug) {
*ptr++ = (char)(rand() % 10);
break;
}
*ptr++ = (char)(rand() & 255);
}
if (verbose) {
if (i && !(i % 32))
printf("\n ");
printf("%u ", data[i]);
}
}
if (verbose)
printf("\n");
checkCudaErrors(cudaMemcpy(gpudata, data, size * sizeof(unsigned), cudaMemcpyHostToDevice));
// So we're now populated and ready to go! We size our launch as
// blocks of up to BLOCKSIZE threads, and appropriate grid size.
// One thread is launched per element.
float elapse;
elapse = run_quicksort_cdp(gpudata, scratchdata, size, NULL);
// run_bitonicsort<SORTTYPE>(gpudata, scratchdata, size, verbose);
checkCudaErrors(cudaDeviceSynchronize());
// Copy back the data and verify correct sort
checkCudaErrors(cudaMemcpy(data, gpudata, size * sizeof(unsigned), cudaMemcpyDeviceToHost));
if (verbose) {
printf("Output: ");
for (unsigned int i = 0; i < size; i++) {
if (i && !(i % 32))
printf("\n ");
printf("%u ", data[i]);
}
printf("\n");
}
unsigned int check;
for (check = 1; check < size; check++) {
if (data[check] < data[check - 1]) {
printf("FAILED at element: %d\n", check);
break;
}
}
if (check != size) {
printf(" cdpAdvancedQuicksort FAILED\n");
exit(EXIT_FAILURE);
}
else
printf(" cdpAdvancedQuicksort PASSED\n");
// Display the time between event recordings
printf("Sorted %u elems in %.3f ms (%.3f Melems/sec)\n", size, elapse, (float)size / (elapse * 1000.0f));
fflush(stdout);
}
// Release everything and we're done
checkCudaErrors(cudaFree(scratchdata));
checkCudaErrors(cudaFree(gpudata));
delete (data);
return 0;
}
static void usage()
{
printf("Syntax: cdpAdvancedQuicksort [-size=<num>] [-seed=<num>] [-debug] "
"[-loop-step=<num>] [-verbose]\n");
printf("If loop_step is non-zero, will run from 1->array_len in steps of "
"loop_step\n");
}
// Host side entry
int main(int argc, char *argv[])
{
int size = 1000000;
unsigned int seed = 0;
int debug = 0;
int loop = 0;
int verbose = 0;
if (checkCmdLineFlag(argc, (const char **)argv, "help") || checkCmdLineFlag(argc, (const char **)argv, "h")) {
usage();
printf("&&&& cdpAdvancedQuicksort WAIVED\n");
exit(EXIT_WAIVED);
}
if (checkCmdLineFlag(argc, (const char **)argv, "size")) {
size = getCmdLineArgumentInt(argc, (const char **)argv, "size");
}
if (checkCmdLineFlag(argc, (const char **)argv, "seed")) {
seed = getCmdLineArgumentInt(argc, (const char **)argv, "seed");
}
if (checkCmdLineFlag(argc, (const char **)argv, "loop-step")) {
loop = getCmdLineArgumentInt(argc, (const char **)argv, "loop-step");
}
if (checkCmdLineFlag(argc, (const char **)argv, "debug")) {
debug = 1;
}
if (checkCmdLineFlag(argc, (const char **)argv, "verbose")) {
verbose = 1;
}
// Get device properties
int cuda_device = findCudaDevice(argc, (const char **)argv);
cudaDeviceProp properties;
checkCudaErrors(cudaGetDeviceProperties(&properties, cuda_device));
int cdpCapable = (properties.major == 3 && properties.minor >= 5) || properties.major >= 4;
printf("GPU device %s has compute capabilities (SM %d.%d)\n", properties.name, properties.major, properties.minor);
if (!cdpCapable) {
printf("cdpAdvancedQuicksort requires SM 3.5 or higher to use CUDA Dynamic "
"Parallelism. Exiting...\n");
exit(EXIT_WAIVED);
}
printf("Running qsort on %d elements with seed %d, on %s\n", size, seed, properties.name);
// Increase the pending kernel launch limit
cudaDeviceSetLimit(cudaLimitDevRuntimePendingLaunchCount, 4096);
run_qsort(size, seed, debug, loop, verbose);
exit(EXIT_SUCCESS);
}

290
cpp/3_CUDA_Features/cdpAdvancedQuicksort/cdpBitonicSort.cu Normal file
View File

@@ -0,0 +1,290 @@
/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of NVIDIA CORPORATION nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
* OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
// This is a basic, recursive bitonic sort taken from
// http://www.iti.fh-flensburg.de/lang/algorithmen/sortieren/bitonic/oddn.htm
//
// The parallel code is based on:
// http://www.tools-of-computing.com/tc/CS/Sorts/bitonic_sort.htm
//
// The multithread code is from me.
#include <cooperative_groups.h>
#include <stdio.h>
namespace cg = cooperative_groups;
#include "cdpQuicksort.h"
// Inline PTX call to return index of highest non-zero bit in a word
static __device__ __forceinline__ unsigned int __btflo(unsigned int word)
{
unsigned int ret;
asm volatile("bfind.u32 %0, %1;" : "=r"(ret) : "r"(word));
return ret;
}
////////////////////////////////////////////////////////////////////////////////
//
// qcompare
//
// Comparison function. Note difference from libc standard in
// that we take by reference, not by pointer. I can't figure
// out how to get a template-to-pointer specialisation working.
// Perhaps it requires a class?
//
////////////////////////////////////////////////////////////////////////////////
__device__ __forceinline__ int qcompare(unsigned &val1, unsigned &val2)
{
return (val1 > val2) ? 1 : (val1 == val2) ? 0 : -1;
}
////////////////////////////////////////////////////////////////////////////////
//
// Basic any-N bitonic sort. We sort "len" elements of "indata", starting
// from the "offset" elements into the input data array. Note that "outdata"
// can safely be the same as "indata" for an in-place sort (we stage through
// shared memory).
//
// We handle non-power-of-2 sizes by padding out to the next largest power of
// 2.
// This is the fully-generic version, for sorting arbitrary data which does not
// have a clear "maximum" value. We track "invalid" entries in a separate array
// to make sure that they always sorts as "max value" elements. A template
// parameter "OOR" allows specialisation to optimise away the invalid tracking.
//
// We can do a more specialised version for int/longlong/flat/double, in which
// we pad out the array with max-value-of-type elements. That's another
// function.
//
// The last step copies from indata -> outdata... the rest is done in-place.
// We use shared memory as temporary storage, which puts an upper limit on
// how much data we can sort per block.
//
////////////////////////////////////////////////////////////////////////////////
static __device__ __forceinline__ void
bitonicsort_kernel(unsigned *indata, unsigned *outdata, unsigned int offset, unsigned int len, cg::thread_block cta)
{
__shared__ unsigned sortbuf[1024]; // Max of 1024 elements - TODO: make this dynamic
// First copy data into shared memory.
unsigned int inside = (threadIdx.x < len);
sortbuf[threadIdx.x] = inside ? indata[threadIdx.x + offset] : 0xffffffffu;
cg::sync(cta);
// Now the sort loops
// Here, "k" is the sort level (remember bitonic does a multi-level butterfly
// style sort)
// and "j" is the partner element in the butterfly.
// Two threads each work on one butterfly, because the read/write needs to
// happen
// simultaneously
for (unsigned int k = 2; k <= blockDim.x; k *= 2) // Butterfly stride increments in powers of 2
{
for (unsigned int j = k >> 1; j > 0; j >>= 1) // Strides also in powers of to, up to <k
{
unsigned int swap_idx = threadIdx.x ^ j; // Index of element we're compare-and-swapping with
unsigned my_elem = sortbuf[threadIdx.x];
unsigned swap_elem = sortbuf[swap_idx];
cg::sync(cta);
// The k'th bit of my threadid (and hence my sort item ID)
// determines if we sort ascending or descending.
// However, since threads are reading from the top AND the bottom of
// the butterfly, if my ID is > swap_idx, then ascending means mine<swap.
// Finally, if either my_elem or swap_elem is out of range, then it
// ALWAYS acts like it's the largest number.
// Confusing? It saves us two writes though.
unsigned int ascend = k * (swap_idx < threadIdx.x);
unsigned int descend = k * (swap_idx > threadIdx.x);
bool swap = false;
if ((threadIdx.x & k) == ascend) {
if (my_elem > swap_elem)
swap = true;
}
if ((threadIdx.x & k) == descend) {
if (my_elem < swap_elem)
swap = true;
}
// If we had to swap, then write my data to the other element's position.
// Don't forget to track out-of-range status too!
if (swap) {
sortbuf[swap_idx] = my_elem;
}
cg::sync(cta);
}
}
// Copy the sorted data from shared memory back to the output buffer
if (threadIdx.x < len)
outdata[threadIdx.x + offset] = sortbuf[threadIdx.x];
}
//////////////////////////////////////////////////////////////////////////////////
// This is an emergency-CTA sort, which sorts an arbitrary sized chunk
// using a single block. Useful for if qsort runs out of nesting depth.
//
// Note that bitonic sort needs enough storage to pad up to the nearest
// power of 2. This means that the double-buffer is always large enough
// (when combined with the main buffer), but we do not get enough space
// to keep OOR information.
//
// This in turn means that this sort does not work with a generic data
// type. It must be a directly-comparable (i.e. with max value) type.
//
////////////////////////////////////////////////////////////////////////////////
static __device__ __forceinline__ void big_bitonicsort_kernel(unsigned *indata,
unsigned *outdata,
unsigned *backbuf,
unsigned int offset,
unsigned int len,
cg::thread_block cta)
{
unsigned int len2 = 1 << (__btflo(len - 1U) + 1); // Round up len to nearest power-of-2
if (threadIdx.x >= len2)
return; // Early out for case where more threads launched than there is
// data
// First, set up our unused values to be the max data type.
for (unsigned int i = len; i < len2; i += blockDim.x) {
unsigned int index = i + threadIdx.x;
if (index < len2) {
// Must split our index between two buffers
if (index < len)
indata[index + offset] = 0xffffffffu;
else
backbuf[index + offset - len] = 0xffffffffu;
}
}
cg::sync(cta);
// Now the sort loops
// Here, "k" is the sort level (remember bitonic does a multi-level butterfly
// style sort)
// and "j" is the partner element in the butterfly.
// Two threads each work on one butterfly, because the read/write needs to
// happen
// simultaneously
for (unsigned int k = 2; k <= len2; k *= 2) // Butterfly stride increments in powers of 2
{
for (unsigned int j = k >> 1; j > 0; j >>= 1) // Strides also in powers of to, up to <k
{
for (unsigned int i = 0; i < len2; i += blockDim.x) {
unsigned int index = threadIdx.x + i;
unsigned int swap_idx = index ^ j; // Index of element we're compare-and-swapping with
// Only do the swap for index<swap_idx (avoids collision between other
// threads)
if (swap_idx > index) {
unsigned my_elem, swap_elem;
if (index < len)
my_elem = indata[index + offset];
else
my_elem = backbuf[index + offset - len];
if (swap_idx < len)
swap_elem = indata[swap_idx + offset];
else
swap_elem = backbuf[swap_idx + offset - len];
// The k'th bit of my index (and hence my sort item ID)
// determines if we sort ascending or descending.
// Also, if either my_elem or swap_elem is out of range, then it
// ALWAYS acts like it's the largest number.
bool swap = false;
if ((index & k) == 0) {
if (my_elem > swap_elem)
swap = true;
}
if ((index & k) == k) {
if (my_elem < swap_elem)
swap = true;
}
// If we had to swap, then write my data to the other element's
// position.
if (swap) {
if (swap_idx < len)
indata[swap_idx + offset] = my_elem;
else
backbuf[swap_idx + offset - len] = my_elem;
if (index < len)
indata[index + offset] = swap_elem;
else
backbuf[index + offset - len] = swap_elem;
}
}
}
cg::sync(cta); // Only need to sync for each "j" pass
}
}
// Copy the sorted data from the input to the output buffer, because we sort
// in-place
if (outdata != indata) {
for (unsigned int i = 0; i < len; i += blockDim.x) {
unsigned int index = i + threadIdx.x;
if (index < len)
outdata[index + offset] = indata[index + offset];
}
}
}
////////////////////////////////////////////////////////////////////////////////
// KERNELS
////////////////////////////////////////////////////////////////////////////////
__global__ void bitonicsort(unsigned *indata, unsigned *outdata, unsigned int offset, unsigned int len)
{
// Handle to thread block group
cg::thread_block cta = cg::this_thread_block();
bitonicsort_kernel(indata, outdata, offset, len, cta);
}
__global__ void
big_bitonicsort(unsigned *indata, unsigned *outdata, unsigned *backbuf, unsigned int offset, unsigned int len)
{
// Handle to thread block group
cg::thread_block cta = cg::this_thread_block();
big_bitonicsort_kernel(indata, outdata, backbuf, offset, len, cta);
}
////////////////////////////////////////////////////////////////////////////////

77
cpp/3_CUDA_Features/cdpAdvancedQuicksort/cdpQuicksort.h Normal file
View File

@@ -0,0 +1,77 @@
/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of NVIDIA CORPORATION nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
* OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
// Definitions for GPU quicksort
#ifndef QUICKSORT_H
#define QUICKSORT_H
#define QSORT_BLOCKSIZE_SHIFT 9
#define QSORT_BLOCKSIZE (1 << QSORT_BLOCKSIZE_SHIFT)
#define BITONICSORT_LEN 1024 // Must be power of 2!
#define QSORT_MAXDEPTH 16 // Will force final bitonic stage at depth QSORT_MAXDEPTH+1
////////////////////////////////////////////////////////////////////////////////
// The algorithm uses several variables updated by using atomic operations.
////////////////////////////////////////////////////////////////////////////////
typedef struct __align__(128) qsortAtomicData_t
{
volatile unsigned int lt_offset; // Current output offset for <pivot
volatile unsigned int gt_offset; // Current output offset for >pivot
volatile unsigned int sorted_count; // Total count sorted, for deciding when
// to launch next wave
volatile unsigned int index; // Ringbuf tracking index. Can be ignored if not using ringbuf.
} qsortAtomicData;
////////////////////////////////////////////////////////////////////////////////
// A ring-buffer for rapid stack allocation
////////////////////////////////////////////////////////////////////////////////
typedef struct qsortRingbuf_t
{
volatile unsigned int head; // Head pointer - we allocate from here
volatile unsigned int tail; // Tail pointer - indicates last still-in-use element
volatile unsigned int count; // Total count allocated
volatile unsigned int max; // Max index allocated
unsigned int stacksize; // Wrap-around size of buffer (must be power of 2)
volatile void *stackbase; // Pointer to the stack we're allocating from
} qsortRingbuf;
// Stack elem count must be power of 2!
#define QSORT_STACK_ELEMS 1 * 1024 * 1024 // One million stack elements is a HUGE number.
__global__ void qsort_warp(unsigned *indata,
unsigned *outdata,
unsigned int len,
qsortAtomicData *atomicData,
qsortRingbuf *ringbuf,
unsigned int source_is_indata,
unsigned int depth);
__global__ void bitonicsort(unsigned *indata, unsigned *outdata, unsigned int offset, unsigned int len);
__global__ void
big_bitonicsort(unsigned *indata, unsigned *outdata, unsigned *backbuf, unsigned int offset, unsigned int len);
#endif // QUICKSORT_H