NASP/cuda-samples
85
0
Fork 0
mirror of https://github.com/NVIDIA/cuda-samples.git synced 2026-10-25 22:28:17 +08:00
Code Issues Packages Projects Releases Wiki Activity

Apply consistent code formatting across the repo. Add clang-format and pre-commit hooks.

Browse Source
This commit is contained in:
Rob Armstrong
2025-03-27 10:30:07 -07:00
parent 2cd58fbc9a
commit ceab6e8bcc
782 changed files with 107230 additions and 106548 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
Samples/3_CUDA_Features/README.md
View File

@@ -76,4 +76,3 @@ This sample demonstrates how using Cooperative Groups (CG) to perform warp aggre
### [graphConditionalNodes](./graphConditionalNodes)
Demonstrate the use of CUDA Graph conditional nodes available starting in CUDA 12.4.

1
Samples/3_CUDA_Features/StreamPriorities/README.md
View File

@@ -32,4 +32,3 @@ Download and install the [CUDA Toolkit 12.5](https://developer.nvidia.com/cuda-d
Make sure the dependencies mentioned in [Dependencies]() section above are installed.
## References (for more details)

244
Samples/3_CUDA_Features/StreamPriorities/StreamPriorities.cu
View File

@@ -30,172 +30,164 @@
// CUDA-C includes
#include <cuda_runtime.h>
#include <helper_cuda.h>
#define TOTAL_SIZE 256 * 1024 * 1024
#define EACH_SIZE 128 * 1024 * 1024
#define EACH_SIZE 128 * 1024 * 1024
// # threadblocks
#define TBLOCKS 1024
#define THREADS 512
// throw error on equality
#define ERR_EQ(X, Y) \
do { \
if ((X) == (Y)) { \
fprintf(stderr, "Error in %s at %s:%d\n", __func__, __FILE__, __LINE__); \
exit(-1); \
} \
} while (0)
#define ERR_EQ(X, Y) \
do { \
if ((X) == (Y)) { \
fprintf(stderr, "Error in %s at %s:%d\n", __func__, __FILE__, __LINE__); \
exit(-1); \
} \
} while (0)
// throw error on difference
#define ERR_NE(X, Y) \
do { \
if ((X) != (Y)) { \
fprintf(stderr, "Error in %s at %s:%d\n", __func__, __FILE__, __LINE__); \
exit(-1); \
} \
} while (0)
#define ERR_NE(X, Y) \
do { \
if ((X) != (Y)) { \
fprintf(stderr, "Error in %s at %s:%d\n", __func__, __FILE__, __LINE__); \
exit(-1); \
} \
} while (0)
// copy from source -> destination arrays
__global__ void memcpy_kernel(int *dst, int *src, size_t n) {
int num = gridDim.x * blockDim.x;
int id = blockDim.x * blockIdx.x + threadIdx.x;
__global__ void memcpy_kernel(int *dst, int *src, size_t n)
{
int num = gridDim.x * blockDim.x;
int id = blockDim.x * blockIdx.x + threadIdx.x;
for (int i = id; i < n / sizeof(int); i += num) {
dst[i] = src[i];
}
for (int i = id; i < n / sizeof(int); i += num) {
dst[i] = src[i];
}
}
// initialise memory
void mem_init(int *buf, size_t n) {
for (int i = 0; i < n / sizeof(int); i++) {
buf[i] = i;
}
void mem_init(int *buf, size_t n)
{
for (int i = 0; i < n / sizeof(int); i++) {
buf[i] = i;
}
}
int main(int argc, char **argv) {
cudaDeviceProp device_prop;
int dev_id;
int main(int argc, char **argv)
{
cudaDeviceProp device_prop;
int dev_id;
printf("Starting [%s]...\n", argv[0]);
printf("Starting [%s]...\n", argv[0]);
// set device
dev_id = findCudaDevice(argc, (const char **)argv);
checkCudaErrors(cudaGetDeviceProperties(&device_prop, dev_id));
// set device
dev_id = findCudaDevice(argc, (const char **)argv);
checkCudaErrors(cudaGetDeviceProperties(&device_prop, dev_id));
if ((device_prop.major << 4) + device_prop.minor < 0x35) {
fprintf(stderr,
"%s requires Compute Capability of SM 3.5 or higher to "
"run.\nexiting...\n",
argv[0]);
exit(EXIT_WAIVED);
}
if ((device_prop.major << 4) + device_prop.minor < 0x35) {
fprintf(stderr,
"%s requires Compute Capability of SM 3.5 or higher to "
"run.\nexiting...\n",
argv[0]);
exit(EXIT_WAIVED);
}
// get the range of priorities available
// [ greatest_priority, lowest_priority ]
int priority_low;
int priority_hi;
checkCudaErrors(
cudaDeviceGetStreamPriorityRange(&priority_low, &priority_hi));
// get the range of priorities available
// [ greatest_priority, lowest_priority ]
int priority_low;
int priority_hi;
checkCudaErrors(cudaDeviceGetStreamPriorityRange(&priority_low, &priority_hi));
printf("CUDA stream priority range: LOW: %d to HIGH: %d\n", priority_low,
priority_hi);
printf("CUDA stream priority range: LOW: %d to HIGH: %d\n", priority_low, priority_hi);
// create streams with highest and lowest available priorities
cudaStream_t st_low;
cudaStream_t st_hi;
checkCudaErrors(cudaStreamCreateWithPriority(&st_low, cudaStreamNonBlocking,
priority_low));
checkCudaErrors(
cudaStreamCreateWithPriority(&st_hi, cudaStreamNonBlocking, priority_hi));
// create streams with highest and lowest available priorities
cudaStream_t st_low;
cudaStream_t st_hi;
checkCudaErrors(cudaStreamCreateWithPriority(&st_low, cudaStreamNonBlocking, priority_low));
checkCudaErrors(cudaStreamCreateWithPriority(&st_hi, cudaStreamNonBlocking, priority_hi));
size_t size;
size = TOTAL_SIZE;
size_t size;
size = TOTAL_SIZE;
// initialise host data
int *h_src_low;
int *h_src_hi;
ERR_EQ(h_src_low = (int *)malloc(size), NULL);
ERR_EQ(h_src_hi = (int *)malloc(size), NULL);
mem_init(h_src_low, size);
mem_init(h_src_hi, size);
// initialise host data
int *h_src_low;
int *h_src_hi;
ERR_EQ(h_src_low = (int *)malloc(size), NULL);
ERR_EQ(h_src_hi = (int *)malloc(size), NULL);
mem_init(h_src_low, size);
mem_init(h_src_hi, size);
// initialise device data
int *h_dst_low;
int *h_dst_hi;
ERR_EQ(h_dst_low = (int *)malloc(size), NULL);
ERR_EQ(h_dst_hi = (int *)malloc(size), NULL);
memset(h_dst_low, 0, size);
memset(h_dst_hi, 0, size);
// initialise device data
int *h_dst_low;
int *h_dst_hi;
ERR_EQ(h_dst_low = (int *)malloc(size), NULL);
ERR_EQ(h_dst_hi = (int *)malloc(size), NULL);
memset(h_dst_low, 0, size);
memset(h_dst_hi, 0, size);
// copy source data -> device
int *d_src_low;
int *d_src_hi;
checkCudaErrors(cudaMalloc(&d_src_low, size));
checkCudaErrors(cudaMalloc(&d_src_hi, size));
checkCudaErrors(
cudaMemcpy(d_src_low, h_src_low, size, cudaMemcpyHostToDevice));
checkCudaErrors(cudaMemcpy(d_src_hi, h_src_hi, size, cudaMemcpyHostToDevice));
// copy source data -> device
int *d_src_low;
int *d_src_hi;
checkCudaErrors(cudaMalloc(&d_src_low, size));
checkCudaErrors(cudaMalloc(&d_src_hi, size));
checkCudaErrors(cudaMemcpy(d_src_low, h_src_low, size, cudaMemcpyHostToDevice));
checkCudaErrors(cudaMemcpy(d_src_hi, h_src_hi, size, cudaMemcpyHostToDevice));
// allocate memory for memcopy destination
int *d_dst_low;
int *d_dst_hi;
checkCudaErrors(cudaMalloc(&d_dst_low, size));
checkCudaErrors(cudaMalloc(&d_dst_hi, size));
// allocate memory for memcopy destination
int *d_dst_low;
int *d_dst_hi;
checkCudaErrors(cudaMalloc(&d_dst_low, size));
checkCudaErrors(cudaMalloc(&d_dst_hi, size));
// create some events
cudaEvent_t ev_start_low;
cudaEvent_t ev_start_hi;
cudaEvent_t ev_end_low;
cudaEvent_t ev_end_hi;
checkCudaErrors(cudaEventCreate(&ev_start_low));
checkCudaErrors(cudaEventCreate(&ev_start_hi));
checkCudaErrors(cudaEventCreate(&ev_end_low));
checkCudaErrors(cudaEventCreate(&ev_end_hi));
// create some events
cudaEvent_t ev_start_low;
cudaEvent_t ev_start_hi;
cudaEvent_t ev_end_low;
cudaEvent_t ev_end_hi;
checkCudaErrors(cudaEventCreate(&ev_start_low));
checkCudaErrors(cudaEventCreate(&ev_start_hi));
checkCudaErrors(cudaEventCreate(&ev_end_low));
checkCudaErrors(cudaEventCreate(&ev_end_hi));
/* */
/* */
// call pair of kernels repeatedly (with different priority streams)
checkCudaErrors(cudaEventRecord(ev_start_low, st_low));
checkCudaErrors(cudaEventRecord(ev_start_hi, st_hi));
// call pair of kernels repeatedly (with different priority streams)
checkCudaErrors(cudaEventRecord(ev_start_low, st_low));
checkCudaErrors(cudaEventRecord(ev_start_hi, st_hi));
for (int i = 0; i < TOTAL_SIZE; i += EACH_SIZE) {
int j = i / sizeof(int);
memcpy_kernel<<<TBLOCKS, THREADS, 0, st_low>>>(d_dst_low + j, d_src_low + j,
EACH_SIZE);
memcpy_kernel<<<TBLOCKS, THREADS, 0, st_hi>>>(d_dst_hi + j, d_src_hi + j,
EACH_SIZE);
}
for (int i = 0; i < TOTAL_SIZE; i += EACH_SIZE) {
int j = i / sizeof(int);
memcpy_kernel<<<TBLOCKS, THREADS, 0, st_low>>>(d_dst_low + j, d_src_low + j, EACH_SIZE);
memcpy_kernel<<<TBLOCKS, THREADS, 0, st_hi>>>(d_dst_hi + j, d_src_hi + j, EACH_SIZE);
}
checkCudaErrors(cudaEventRecord(ev_end_low, st_low));
checkCudaErrors(cudaEventRecord(ev_end_hi, st_hi));
checkCudaErrors(cudaEventRecord(ev_end_low, st_low));
checkCudaErrors(cudaEventRecord(ev_end_hi, st_hi));
checkCudaErrors(cudaEventSynchronize(ev_end_low));
checkCudaErrors(cudaEventSynchronize(ev_end_hi));
checkCudaErrors(cudaEventSynchronize(ev_end_low));
checkCudaErrors(cudaEventSynchronize(ev_end_hi));
/* */
/* */
size = TOTAL_SIZE;
checkCudaErrors(
cudaMemcpy(h_dst_low, d_dst_low, size, cudaMemcpyDeviceToHost));
checkCudaErrors(cudaMemcpy(h_dst_hi, d_dst_hi, size, cudaMemcpyDeviceToHost));
size = TOTAL_SIZE;
checkCudaErrors(cudaMemcpy(h_dst_low, d_dst_low, size, cudaMemcpyDeviceToHost));
checkCudaErrors(cudaMemcpy(h_dst_hi, d_dst_hi, size, cudaMemcpyDeviceToHost));
// check results of kernels
ERR_NE(memcmp(h_dst_low, h_src_low, size), 0);
ERR_NE(memcmp(h_dst_hi, h_src_hi, size), 0);
// check results of kernels
ERR_NE(memcmp(h_dst_low, h_src_low, size), 0);
ERR_NE(memcmp(h_dst_hi, h_src_hi, size), 0);
// check timings
float ms_low;
float ms_hi;
checkCudaErrors(cudaEventElapsedTime(&ms_low, ev_start_low, ev_end_low));
checkCudaErrors(cudaEventElapsedTime(&ms_hi, ev_start_hi, ev_end_hi));
// check timings
float ms_low;
float ms_hi;
checkCudaErrors(cudaEventElapsedTime(&ms_low, ev_start_low, ev_end_low));
checkCudaErrors(cudaEventElapsedTime(&ms_hi, ev_start_hi, ev_end_hi));
printf("elapsed time of kernels launched to LOW priority stream: %.3lf ms\n",
ms_low);
printf("elapsed time of kernels launched to HI priority stream: %.3lf ms\n",
ms_hi);
printf("elapsed time of kernels launched to LOW priority stream: %.3lf ms\n", ms_low);
printf("elapsed time of kernels launched to HI priority stream: %.3lf ms\n", ms_hi);
exit(EXIT_SUCCESS);
exit(EXIT_SUCCESS);
}

330
Samples/3_CUDA_Features/bf16TensorCoreGemm/bf16TensorCoreGemm.cu
View File

@@ -58,15 +58,15 @@
// but carefully enough to avoid local memory use.
#include <assert.h>
#include <stdio.h>
#include <cuda.h>
#include <cuda/pipeline>
#include <cuda_bf16.h>
#include <mma.h>
#include <cuda/pipeline>
#include <stdio.h>
// helper functions and utilities to work with CUDA
#include <helper_functions.h>
#include <helper_cuda.h>
#include <helper_functions.h>
// Externally configurable parameters.
@@ -107,7 +107,7 @@
// Implementation constants.
#define WARPS_PER_BLOCK 8
#define WARPS_PER_BLOCK 8
#define THREADS_PER_BLOCK (WARP_SIZE * WARPS_PER_BLOCK)
#if SHARED_MEMORY_LIMIT_64K
@@ -125,10 +125,10 @@
#define CHUNK_K 8
#endif
#define CHUNK_LINE_BYTES (CHUNK_K * K * sizeof(__nv_bfloat16))
#define WARP_COPY_BYTES (WARP_SIZE * sizeof(int4))
#define CHUNK_LINE_BYTES (CHUNK_K * K * sizeof(__nv_bfloat16))
#define WARP_COPY_BYTES (WARP_SIZE * sizeof(int4))
#define CHUNK_COPY_LINES_PER_WARP (WARP_COPY_BYTES / CHUNK_LINE_BYTES)
#define CHUNK_COPY_LINE_LANES (WARP_SIZE / CHUNK_COPY_LINES_PER_WARP)
#define CHUNK_COPY_LINE_LANES (WARP_SIZE / CHUNK_COPY_LINES_PER_WARP)
#define BLOCK_ROW_WARPS 2
#define BLOCK_COL_WARPS 4
@@ -158,25 +158,24 @@
// we must keep each row and column 256-bit aligned, as required by nvcuda::wmma::load_matrix_sync.
#define SKEW_BF16 16
#define checkKernelErrors(expr) do { \
expr; \
\
cudaError_t __err = cudaGetLastError(); \
if (__err != cudaSuccess) { \
printf("Line %d: '%s' failed: %s\n", __LINE__, # expr, cudaGetErrorString(__err)); \
abort(); \
} \
} while(0)
#define checkKernelErrors(expr) \
do { \
expr; \
\
cudaError_t __err = cudaGetLastError(); \
if (__err != cudaSuccess) { \
printf("Line %d: '%s' failed: %s\n", __LINE__, #expr, cudaGetErrorString(__err)); \
abort(); \
} \
} while (0)
enum kernels
{
bf16mma_shmem_gemm_async_copy = 0, // __nv_bfloat16 MMA shmem using kernel with async_copy
bf16mma_shmem_gemm = 1, // __nv_bfloat16 MMA shmem using kernel normal copy (without async_copy).
simple_bf16mma_gemm = 2 // __nv_bfloat16 MMA non-shmem using simple kernel.
enum kernels {
bf16mma_shmem_gemm_async_copy = 0, // __nv_bfloat16 MMA shmem using kernel with async_copy
bf16mma_shmem_gemm = 1, // __nv_bfloat16 MMA shmem using kernel normal copy (without async_copy).
simple_bf16mma_gemm = 2 // __nv_bfloat16 MMA non-shmem using simple kernel.
};
const char* kernelNames[] = {"compute_bf16gemm_async_copy", "compute_bf16gemm",
"simple_wmma_bf16gemm"};
const char *kernelNames[] = {"compute_bf16gemm_async_copy", "compute_bf16gemm", "simple_wmma_bf16gemm"};
using namespace nvcuda;
@@ -184,22 +183,23 @@ __host__ void init_host_matrices(__nv_bfloat16 *a, __nv_bfloat16 *b, float *c)
{
for (int i = 0; i < M_GLOBAL; i++) {
for (int j = 0; j < K_GLOBAL; j++) {
a[i*K_GLOBAL+j] = (__nv_bfloat16)(float)(rand() % 3);
a[i * K_GLOBAL + j] = (__nv_bfloat16)(float)(rand() % 3);
}
}
for (int i = 0; i < N_GLOBAL; i++) {
for (int j = 0; j < K_GLOBAL; j++) {
b[i*K_GLOBAL+j] = (__nv_bfloat16)(float)(rand() % 3);
b[i * K_GLOBAL + j] = (__nv_bfloat16)(float)(rand() % 3);
}
}
for (int t = 0; t < M_GLOBAL * N_GLOBAL; t++) {
c[t] = (float)(rand() % 3);
c[t] = (float)(rand() % 3);
}
}
__global__ void compute_bf16gemm(const __nv_bfloat16 *A, const __nv_bfloat16 *B, const float *C, float *D, float alpha, float beta)
__global__ void
compute_bf16gemm(const __nv_bfloat16 *A, const __nv_bfloat16 *B, const float *C, float *D, float alpha, float beta)
{
#if __CUDA_ARCH__ >= 800
extern __shared__ __nv_bfloat16 shmem[][CHUNK_K * K + SKEW_BF16];
@@ -212,10 +212,11 @@ __global__ void compute_bf16gemm(const __nv_bfloat16 *A, const __nv_bfloat16 *B,
const size_t shmem_idx_b_off = BLOCK_COL_TILES * M;
// This pointer is used to access the C and D matrix tiles this warp computes.
float *shmem_warp_tile_ptr = (float*)&shmem[0][0] + (warpId / BLOCK_ROW_WARPS) * SHMEM_STRIDE * N * BLOCK_ROW_WARPS + (warpId % BLOCK_ROW_WARPS) * SHMEM_OFFSET;
float *shmem_warp_tile_ptr = (float *)&shmem[0][0] + (warpId / BLOCK_ROW_WARPS) * SHMEM_STRIDE * N * BLOCK_ROW_WARPS
+ (warpId % BLOCK_ROW_WARPS) * SHMEM_OFFSET;
// This pointer is used to stream the C and D matrices block-wide tile to and from shared memory.
float *shmem_warp_stream_ptr = (float*)&shmem[0][0] + warpId * SHMEM_STRIDE * N;
float *shmem_warp_stream_ptr = (float *)&shmem[0][0] + warpId * SHMEM_STRIDE * N;
// Adjust the beta scaler, as it'll be multiplied by alpha at the end of
// each tile computation. Technically this is not generally correct (may result
@@ -225,7 +226,7 @@ __global__ void compute_bf16gemm(const __nv_bfloat16 *A, const __nv_bfloat16 *B,
// Each CTA slides along the 128 x 128 tiles from the top left corner of the matrix to the
// right and down, and selects the next tile to compute. Once there's no such tile,
// all warps in this CTA exit.
for(unsigned int block_pos = blockIdx.x;; block_pos += gridDim.x) {
for (unsigned int block_pos = blockIdx.x;; block_pos += gridDim.x) {
const unsigned int block_tile_i = ((block_pos * BLOCK_ROW_TILES) / N_TILES) * (BLOCK_COL_TILES);
const unsigned int block_tile_j = (block_pos * BLOCK_COL_TILES) % N_TILES;
@@ -235,14 +236,14 @@ __global__ void compute_bf16gemm(const __nv_bfloat16 *A, const __nv_bfloat16 *B,
}
// This warp's pointer to the C matrix data to copy memory from to shared memory.
const size_t gmem_idx = (block_tile_i + warpId) * M * GLOBAL_MEM_STRIDE + block_tile_j * N;
const size_t gmem_idx = (block_tile_i + warpId) * M * GLOBAL_MEM_STRIDE + block_tile_j * N;
const float *src_gmem_warp_stream_ptr = &C[gmem_idx];
// Stream multiple C tiles to shared memory.
#pragma unroll
for (int i = 0; i < N; i++) {
*((int4*)(shmem_warp_stream_ptr + SHMEM_STRIDE * i) + laneId) =
*((int4*)(src_gmem_warp_stream_ptr + GLOBAL_MEM_STRIDE * i) + laneId);
*((int4 *)(shmem_warp_stream_ptr + SHMEM_STRIDE * i) + laneId) =
*((int4 *)(src_gmem_warp_stream_ptr + GLOBAL_MEM_STRIDE * i) + laneId);
}
__syncthreads();
@@ -266,7 +267,7 @@ __global__ void compute_bf16gemm(const __nv_bfloat16 *A, const __nv_bfloat16 *B,
// Scale the C matrix.
#pragma unroll
for (int i = 0; i < WARP_COL_TILES; i++) {
for (int i = 0; i < WARP_COL_TILES; i++) {
#pragma unroll
for (int j = 0; j < WARP_ROW_TILES; j++) {
#pragma unroll
@@ -278,16 +279,19 @@ __global__ void compute_bf16gemm(const __nv_bfloat16 *A, const __nv_bfloat16 *B,
// Select what warp copies what matrix to shared memory.
// Warps 0-3 copy the A matrix, warps 4-7 copy the B matrix.
const __nv_bfloat16 *warp_ptr = (warpId < (WARPS_PER_BLOCK/2)) ? (&A[block_tile_i * M * K_GLOBAL] + M * K_GLOBAL * (warpId % (WARPS_PER_BLOCK/2)) * 2) :
(&B[block_tile_j * N * K_GLOBAL] + N * K_GLOBAL * (warpId % (WARPS_PER_BLOCK/2)) * 2);
const __nv_bfloat16 *warp_ptr =
(warpId < (WARPS_PER_BLOCK / 2))
? (&A[block_tile_i * M * K_GLOBAL] + M * K_GLOBAL * (warpId % (WARPS_PER_BLOCK / 2)) * 2)
: (&B[block_tile_j * N * K_GLOBAL] + N * K_GLOBAL * (warpId % (WARPS_PER_BLOCK / 2)) * 2);
// Go through the global K dimension by a fixed step at a time.
#pragma unroll
for (int tile_k = 0; tile_k < K_TILES; tile_k += CHUNK_K) {
// Copy slices of the A and B matrices to shared memory.
// The first half of the warps in the CTA copy the A matrix, the rest copy the B matrix.
size_t shmem_idx = warpId < (WARPS_PER_BLOCK/2) ? (M * (warpId % (WARPS_PER_BLOCK/2)) * 2) :
(N * (warpId % (WARPS_PER_BLOCK/2)) * 2 + shmem_idx_b_off);
size_t shmem_idx = warpId < (WARPS_PER_BLOCK / 2)
? (M * (warpId % (WARPS_PER_BLOCK / 2)) * 2)
: (N * (warpId % (WARPS_PER_BLOCK / 2)) * 2 + shmem_idx_b_off);
// First half of the warp copies the first row / column of the matrix,
// the second half of the warp copies the next.
@@ -297,9 +301,10 @@ __global__ void compute_bf16gemm(const __nv_bfloat16 *A, const __nv_bfloat16 *B,
shmem_idx += laneId / CHUNK_COPY_LINE_LANES;
#pragma unroll
for(int i = 0; i < ((WARP_SIZE/2) / CHUNK_COPY_LINES_PER_WARP) * 2; i++) {
for (int i = 0; i < ((WARP_SIZE / 2) / CHUNK_COPY_LINES_PER_WARP) * 2; i++) {
// Copy 16 bytes at once in each lane.
*((int4*)&shmem[shmem_idx][0] + (laneId % CHUNK_COPY_LINE_LANES)) = *((int4*)lane_ptr + (laneId % CHUNK_COPY_LINE_LANES));
*((int4 *)&shmem[shmem_idx][0] + (laneId % CHUNK_COPY_LINE_LANES)) =
*((int4 *)lane_ptr + (laneId % CHUNK_COPY_LINE_LANES));
// Advance the global memory pointer and the shared memory index.
lane_ptr = lane_ptr + K_GLOBAL * CHUNK_COPY_LINES_PER_WARP;
@@ -316,8 +321,8 @@ __global__ void compute_bf16gemm(const __nv_bfloat16 *A, const __nv_bfloat16 *B,
#pragma unroll
for (int i = 0; i < WARP_COL_TILES; i++) {
size_t shmem_idx_a = (warpId/BLOCK_ROW_WARPS) * M * BLOCK_ROW_WARPS + (i * M);
const __nv_bfloat16 *tile_ptr = &shmem[shmem_idx_a][k_step * K];
size_t shmem_idx_a = (warpId / BLOCK_ROW_WARPS) * M * BLOCK_ROW_WARPS + (i * M);
const __nv_bfloat16 *tile_ptr = &shmem[shmem_idx_a][k_step * K];
wmma::load_matrix_sync(a[i], tile_ptr, K * CHUNK_K + SKEW_BF16);
@@ -326,7 +331,7 @@ __global__ void compute_bf16gemm(const __nv_bfloat16 *A, const __nv_bfloat16 *B,
if (i == 0) {
// Load the B matrix fragment once, because it is going to be reused
// against the other A matrix fragments.
size_t shmem_idx_b = shmem_idx_b_off + (WARP_ROW_TILES * N) * (warpId%2) + (j * N);
size_t shmem_idx_b = shmem_idx_b_off + (WARP_ROW_TILES * N) * (warpId % 2) + (j * N);
const __nv_bfloat16 *tile_ptr = &shmem[shmem_idx_b][k_step * K];
wmma::load_matrix_sync(b[j], tile_ptr, K * CHUNK_K + SKEW_BF16);
@@ -364,8 +369,8 @@ __global__ void compute_bf16gemm(const __nv_bfloat16 *A, const __nv_bfloat16 *B,
#pragma unroll
for (int i = 0; i < N; i++) {
*((float4*)(dst_gmem_warp_stream_ptr + GLOBAL_MEM_STRIDE * i) + laneId) =
*((float4*)(shmem_warp_stream_ptr + SHMEM_STRIDE * i) + laneId);
*((float4 *)(dst_gmem_warp_stream_ptr + GLOBAL_MEM_STRIDE * i) + laneId) =
*((float4 *)(shmem_warp_stream_ptr + SHMEM_STRIDE * i) + laneId);
}
__syncthreads();
@@ -373,7 +378,12 @@ __global__ void compute_bf16gemm(const __nv_bfloat16 *A, const __nv_bfloat16 *B,
#endif
}
__global__ void compute_bf16gemm_async_copy(const __nv_bfloat16 *A, const __nv_bfloat16 *B, const float *C, float *D, float alpha, float beta)
__global__ void compute_bf16gemm_async_copy(const __nv_bfloat16 *A,
const __nv_bfloat16 *B,
const float *C,
float *D,
float alpha,
float beta)
{
#if __CUDA_ARCH__ >= 800
extern __shared__ __nv_bfloat16 shmem[][CHUNK_K * K + SKEW_BF16];
@@ -386,24 +396,25 @@ __global__ void compute_bf16gemm_async_copy(const __nv_bfloat16 *A, const __nv_b
constexpr size_t shmem_idx_b_off = BLOCK_COL_TILES * M;
// This pointer is used to access the C and D matrix tiles this warp computes.
float *shmem_warp_tile_ptr = (float*)&shmem[0][0] + (warpId / BLOCK_ROW_WARPS) * SHMEM_STRIDE * N * BLOCK_ROW_WARPS + (warpId % BLOCK_ROW_WARPS) * SHMEM_OFFSET;
float *shmem_warp_tile_ptr = (float *)&shmem[0][0] + (warpId / BLOCK_ROW_WARPS) * SHMEM_STRIDE * N * BLOCK_ROW_WARPS
+ (warpId % BLOCK_ROW_WARPS) * SHMEM_OFFSET;
// This pointer is used to stream the C and D matrices block-wide tile to and from shared memory.
float *shmem_warp_stream_ptr = (float*)&shmem[0][0] + warpId * SHMEM_STRIDE * N;
float *shmem_warp_stream_ptr = (float *)&shmem[0][0] + warpId * SHMEM_STRIDE * N;
// Adjust the beta scaler, as it'll be multiplied by alpha at the end of
// each tile computation. Technically this is not generally correct (may result
// in a loss of precision). Zero still needs to be specially handled though.
beta /= alpha;
cuda::pipeline<cuda::thread_scope_thread> pipe = cuda::make_pipeline();
const auto shape4 = cuda::aligned_size_t<alignof(float4)>(sizeof(float4));
constexpr int loadStride = 2; // load 4 floats, left-shift by 2.
cuda::pipeline<cuda::thread_scope_thread> pipe = cuda::make_pipeline();
const auto shape4 = cuda::aligned_size_t<alignof(float4)>(sizeof(float4));
constexpr int loadStride = 2; // load 4 floats, left-shift by 2.
// Each CTA slides along the 128 x 128 tiles from the top left corner of the matrix to the
// right and down, and selects the next tile to compute. Once there's no such tile,
// all warps in this CTA exit.
for(unsigned int block_pos = blockIdx.x;; block_pos += gridDim.x) {
for (unsigned int block_pos = blockIdx.x;; block_pos += gridDim.x) {
const unsigned int block_tile_i = ((block_pos * BLOCK_ROW_TILES) / N_TILES) * (BLOCK_COL_TILES);
const unsigned int block_tile_j = (block_pos * BLOCK_COL_TILES) % N_TILES;
@@ -413,7 +424,7 @@ __global__ void compute_bf16gemm_async_copy(const __nv_bfloat16 *A, const __nv_b
}
// This warp's pointer to the C matrix data to copy memory from to shared memory.
const size_t gmem_idx = (block_tile_i + warpId) * M * GLOBAL_MEM_STRIDE + block_tile_j * N;
const size_t gmem_idx = (block_tile_i + warpId) * M * GLOBAL_MEM_STRIDE + block_tile_j * N;
const float *src_gmem_warp_stream_ptr = &C[gmem_idx];
// Stream multiple C tiles to shared memory.
@@ -422,8 +433,9 @@ __global__ void compute_bf16gemm_async_copy(const __nv_bfloat16 *A, const __nv_b
pipe.producer_acquire();
cuda::memcpy_async(&shmem_warp_stream_ptr[(SHMEM_STRIDE * i) + (laneId << loadStride)],
&src_gmem_warp_stream_ptr[(GLOBAL_MEM_STRIDE * i) + (laneId << loadStride)],
shape4, pipe);
&src_gmem_warp_stream_ptr[(GLOBAL_MEM_STRIDE * i) + (laneId << loadStride)],
shape4,
pipe);
pipe.producer_commit();
}
@@ -458,13 +470,15 @@ __global__ void compute_bf16gemm_async_copy(const __nv_bfloat16 *A, const __nv_b
__syncthreads();
// Select what warp copies what matrix to shared memory.
// Warps 0-3 copy the A matrix, warps 4-7 copy the B matrix.
const __nv_bfloat16 *warp_ptr = (warpId < (WARPS_PER_BLOCK/2)) ? (&A[block_tile_i * M * K_GLOBAL] + M * K_GLOBAL * (warpId % (WARPS_PER_BLOCK/2)) * 2) :
(&B[block_tile_j * N * K_GLOBAL] + N * K_GLOBAL * (warpId % (WARPS_PER_BLOCK/2)) * 2);
const __nv_bfloat16 *warp_ptr =
(warpId < (WARPS_PER_BLOCK / 2))
? (&A[block_tile_i * M * K_GLOBAL] + M * K_GLOBAL * (warpId % (WARPS_PER_BLOCK / 2)) * 2)
: (&B[block_tile_j * N * K_GLOBAL] + N * K_GLOBAL * (warpId % (WARPS_PER_BLOCK / 2)) * 2);
constexpr int chunksPerLane = ((WARP_SIZE/2) / CHUNK_COPY_LINES_PER_WARP) * 2;
constexpr int chunksPerLane = ((WARP_SIZE / 2) / CHUNK_COPY_LINES_PER_WARP) * 2;
constexpr int loadStrideBfloat8 = 3; // load 8 bfloats, left-shift by 3.
const int laneLoadElem = (laneId % CHUNK_COPY_LINE_LANES) << loadStrideBfloat8;
const int stridePerLaneCopy = (laneId / CHUNK_COPY_LINE_LANES);
const int laneLoadElem = (laneId % CHUNK_COPY_LINE_LANES) << loadStrideBfloat8;
const int stridePerLaneCopy = (laneId / CHUNK_COPY_LINE_LANES);
// Go through the global K dimension by a fixed step at a time.
#pragma unroll
@@ -472,7 +486,8 @@ __global__ void compute_bf16gemm_async_copy(const __nv_bfloat16 *A, const __nv_b
// Copy slices of the A and B matrices to shared memory.
// The first half of the warps in the CTA copy the A matrix, the rest copy the B matrix.
// As for bf16 MMA M == N we use M for warp 4-7 + shmem_idx_b_off.
size_t shmem_idx = (M * (warpId % (WARPS_PER_BLOCK/2)) * 2) + ((warpId / (WARPS_PER_BLOCK/2)) * shmem_idx_b_off);
size_t shmem_idx =
(M * (warpId % (WARPS_PER_BLOCK / 2)) * 2) + ((warpId / (WARPS_PER_BLOCK / 2)) * shmem_idx_b_off);
// First half of the warp copies the first row / column of the matrix,
// the second half of the warp copies the next.
@@ -482,7 +497,7 @@ __global__ void compute_bf16gemm_async_copy(const __nv_bfloat16 *A, const __nv_b
shmem_idx += stridePerLaneCopy;
#pragma unroll
for(int i = 0; i < chunksPerLane; i++) {
for (int i = 0; i < chunksPerLane; i++) {
// Copy 16 bytes at once in each lane.
pipe.producer_acquire();
cuda::memcpy_async(&shmem[shmem_idx][laneLoadElem], lane_ptr, shape4, pipe);
@@ -502,8 +517,8 @@ __global__ void compute_bf16gemm_async_copy(const __nv_bfloat16 *A, const __nv_b
#pragma unroll
for (int i = 0; i < WARP_COL_TILES; i++) {
size_t shmem_idx_a = (warpId / BLOCK_ROW_WARPS) * M * BLOCK_ROW_WARPS + (i * M);
const __nv_bfloat16 *tile_ptr = &shmem[shmem_idx_a][k_step * K];
size_t shmem_idx_a = (warpId / BLOCK_ROW_WARPS) * M * BLOCK_ROW_WARPS + (i * M);
const __nv_bfloat16 *tile_ptr = &shmem[shmem_idx_a][k_step * K];
wmma::load_matrix_sync(a[i], tile_ptr, K * CHUNK_K + SKEW_BF16);
@@ -512,7 +527,7 @@ __global__ void compute_bf16gemm_async_copy(const __nv_bfloat16 *A, const __nv_b
if (i == 0) {
// Load the B matrix fragment once, because it is going to be reused
// against the other A matrix fragments.
size_t shmem_idx_b = shmem_idx_b_off + (WARP_ROW_TILES * N) * (warpId%2) + (j * N);
size_t shmem_idx_b = shmem_idx_b_off + (WARP_ROW_TILES * N) * (warpId % 2) + (j * N);
const __nv_bfloat16 *tile_ptr = &shmem[shmem_idx_b][k_step * K];
wmma::load_matrix_sync(b[j], tile_ptr, K * CHUNK_K + SKEW_BF16);
@@ -551,8 +566,8 @@ __global__ void compute_bf16gemm_async_copy(const __nv_bfloat16 *A, const __nv_b
#pragma unroll
for (int i = 0; i < N; i++) {
*((int4*)(dst_gmem_warp_stream_ptr + GLOBAL_MEM_STRIDE * i) + laneId) =
*((int4*)(shmem_warp_stream_ptr + SHMEM_STRIDE * i) + laneId);
*((int4 *)(dst_gmem_warp_stream_ptr + GLOBAL_MEM_STRIDE * i) + laneId) =
*((int4 *)(shmem_warp_stream_ptr + SHMEM_STRIDE * i) + laneId);
}
__syncthreads();
@@ -562,73 +577,86 @@ __global__ void compute_bf16gemm_async_copy(const __nv_bfloat16 *A, const __nv_b
// Performs an MxNxK bf16 GEMM (C=alpha*A*B + beta*C) assuming:
// 1) Matrices are packed in memory.
// 2) M, N and K are multiples of 16, 16 and 16 respectively.
// 2) M, N and K are multiples of 16, 16 and 16 respectively.
// 3) A is row major, B is column major matrix.
// Note: This is a less performant version of the compute_bf16gemm kernel. It is designed for
// demonstration purposes only to show the CUDA WMMA API use without relying on
// availability of the shared memory.
__global__ void simple_wmma_bf16gemm(__nv_bfloat16 *a, __nv_bfloat16 *b, float *c, float *d, int m_ld, int n_ld, int k_ld, float alpha, float beta)
__global__ void simple_wmma_bf16gemm(__nv_bfloat16 *a,
__nv_bfloat16 *b,
float *c,
float *d,
int m_ld,
int n_ld,
int k_ld,
float alpha,
float beta)
{
#if __CUDA_ARCH__ >= 800
// Leading dimensions. Packed with no transpositions.
// Leading dimensions. Packed with no transpositions.
int lda = k_ld;
int ldb = k_ld;
int ldc = n_ld;
// Tile using a 2D grid
int warpM = (blockIdx.x * blockDim.x + threadIdx.x) / warpSize;
int warpN = (blockIdx.y * blockDim.y + threadIdx.y);
// Declare the fragments
wmma::fragment<wmma::matrix_a, M, N, K, __nv_bfloat16, wmma::row_major> a_frag;
wmma::fragment<wmma::matrix_b, M, N, K, __nv_bfloat16, wmma::col_major> b_frag;
wmma::fragment<wmma::accumulator, M, N, K, float> acc_frag;
wmma::fragment<wmma::accumulator, M, N, K, float> c_frag;
// Tile using a 2D grid
int warpM = (blockIdx.x * blockDim.x + threadIdx.x) / warpSize;
int warpN = (blockIdx.y * blockDim.y + threadIdx.y);
wmma::fill_fragment(acc_frag, 0.0f);
// Declare the fragments
wmma::fragment<wmma::matrix_a, M, N, K, __nv_bfloat16, wmma::row_major> a_frag;
wmma::fragment<wmma::matrix_b, M, N, K, __nv_bfloat16, wmma::col_major> b_frag;
wmma::fragment<wmma::accumulator, M, N, K, float> acc_frag;
wmma::fragment<wmma::accumulator, M, N, K, float> c_frag;
// Loop over k
for (int i = 0; i < k_ld; i += K) {
int aCol = i;
int aRow = warpM * M;
wmma::fill_fragment(acc_frag, 0.0f);
int bCol = i;
int bRow = warpN * N;
// Loop over k
for (int i = 0; i < k_ld; i += K) {
int aCol = i;
int aRow = warpM * M;
// Bounds checking
if (aRow < m_ld && aCol < k_ld && bRow < k_ld && bCol < n_ld) {
// Load the inputs
wmma::load_matrix_sync(a_frag, a + aCol + aRow * lda, lda);
wmma::load_matrix_sync(b_frag, b + bRow + bCol * ldb, ldb);
// Perform the matrix multiplication
wmma::mma_sync(acc_frag, a_frag, b_frag, acc_frag);
int bCol = i;
int bRow = warpN * N;
}
}
// Bounds checking
if (aRow < m_ld && aCol < k_ld && bRow < k_ld && bCol < n_ld) {
// Load the inputs
wmma::load_matrix_sync(a_frag, a + aCol + aRow * lda, lda);
wmma::load_matrix_sync(b_frag, b + bRow + bCol * ldb, ldb);
// Load in the current value of c, scale it by beta, and add this our result scaled by alpha
int cCol = warpN * N;
int cRow = warpM * M;
// Perform the matrix multiplication
wmma::mma_sync(acc_frag, a_frag, b_frag, acc_frag);
}
}
if (cRow < m_ld && cCol < n_ld) {
wmma::load_matrix_sync(c_frag, c + cCol + cRow * ldc, ldc, wmma::mem_row_major);
// Load in the current value of c, scale it by beta, and add this our result scaled by alpha
int cCol = warpN * N;
int cRow = warpM * M;
for(int i=0; i < c_frag.num_elements; i++) {
c_frag.x[i] = alpha * acc_frag.x[i] + beta * c_frag.x[i];
}
if (cRow < m_ld && cCol < n_ld) {
wmma::load_matrix_sync(c_frag, c + cCol + cRow * ldc, ldc, wmma::mem_row_major);
// Store the output
wmma::store_matrix_sync(d + cCol + cRow * ldc, c_frag, ldc, wmma::mem_row_major);
}
for (int i = 0; i < c_frag.num_elements; i++) {
c_frag.x[i] = alpha * acc_frag.x[i] + beta * c_frag.x[i];
}
// Store the output
wmma::store_matrix_sync(d + cCol + cRow * ldc, c_frag, ldc, wmma::mem_row_major);
}
#endif
}
__host__ void matMultiplyOnHost(__nv_bfloat16 *A, __nv_bfloat16 *B, float *C,
float alpha, float beta,
int numARows, int numAColumns,
int numBRows, int numBColumns,
int numCRows, int numCColumns)
__host__ void matMultiplyOnHost(__nv_bfloat16 *A,
__nv_bfloat16 *B,
float *C,
float alpha,
float beta,
int numARows,
int numAColumns,
int numBRows,
int numBColumns,
int numCRows,
int numCColumns)
{
for (int i = 0; i < numCRows; i++) {
for (int j = 0; j < numCColumns; j++) {
@@ -638,7 +666,7 @@ __host__ void matMultiplyOnHost(__nv_bfloat16 *A, __nv_bfloat16 *B, float *C,
temp += (float)A[i * numAColumns + k] * (float)B[j * numBRows + k];
}
C[i*numCColumns + j] = temp * alpha + beta * C[i * numCColumns + j];
C[i * numCColumns + j] = temp * alpha + beta * C[i * numCColumns + j];
}
}
}
@@ -664,29 +692,29 @@ int main(int argc, char **argv)
__nv_bfloat16 *A_h = NULL;
__nv_bfloat16 *B_h = NULL;
float *C_h = NULL;
float *C_h = NULL;
#if CPU_DEBUG
float *result_hD = NULL;
float *result_hD = NULL;
float *result_host = NULL;
#endif
A_h = (__nv_bfloat16*) malloc(sizeof(__nv_bfloat16) * M_GLOBAL * K_GLOBAL);
B_h = (__nv_bfloat16*) malloc(sizeof(__nv_bfloat16) * K_GLOBAL * N_GLOBAL);
C_h = (float*) malloc(sizeof(float) * M_GLOBAL * N_GLOBAL);
A_h = (__nv_bfloat16 *)malloc(sizeof(__nv_bfloat16) * M_GLOBAL * K_GLOBAL);
B_h = (__nv_bfloat16 *)malloc(sizeof(__nv_bfloat16) * K_GLOBAL * N_GLOBAL);
C_h = (float *)malloc(sizeof(float) * M_GLOBAL * N_GLOBAL);
#if CPU_DEBUG
result_hD = (float*) malloc(sizeof(float) * M_GLOBAL * N_GLOBAL);
result_host = (float*) malloc(sizeof(float) * M_GLOBAL * N_GLOBAL);
result_hD = (float *)malloc(sizeof(float) * M_GLOBAL * N_GLOBAL);
result_host = (float *)malloc(sizeof(float) * M_GLOBAL * N_GLOBAL);
#endif
__nv_bfloat16 *A = NULL;
__nv_bfloat16 *B = NULL;
float *C = NULL;
float *D = NULL;
float *C = NULL;
float *D = NULL;
checkCudaErrors(cudaMalloc((void**)&A, sizeof(__nv_bfloat16) * M_GLOBAL * K_GLOBAL));
checkCudaErrors(cudaMalloc((void**)&B, sizeof(__nv_bfloat16) * N_GLOBAL * K_GLOBAL));
checkCudaErrors(cudaMalloc((void**)&C, sizeof(float) * M_GLOBAL * N_GLOBAL));
checkCudaErrors(cudaMalloc((void**)&D, sizeof(float) * M_GLOBAL * N_GLOBAL));
checkCudaErrors(cudaMalloc((void **)&A, sizeof(__nv_bfloat16) * M_GLOBAL * K_GLOBAL));
checkCudaErrors(cudaMalloc((void **)&B, sizeof(__nv_bfloat16) * N_GLOBAL * K_GLOBAL));
checkCudaErrors(cudaMalloc((void **)&C, sizeof(float) * M_GLOBAL * N_GLOBAL));
checkCudaErrors(cudaMalloc((void **)&D, sizeof(float) * M_GLOBAL * N_GLOBAL));
assert(((unsigned long long)A) % 128 == 0);
assert(((unsigned long long)B) % 128 == 0);
@@ -714,11 +742,11 @@ int main(int argc, char **argv)
printf("Required shared memory size: %lu Kb\n", SHMEM_SZ / 1024UL);
const float alpha = 1.1f;
const float beta = 1.2f;
const float beta = 1.2f;
cudaEvent_t start, stop;
checkCudaErrors(cudaEventCreate(&start));
checkCudaErrors(cudaEventCreate(&start));
checkCudaErrors(cudaEventCreate(&stop));
checkCudaErrors(cudaEventRecord(start));
@@ -740,26 +768,30 @@ int main(int argc, char **argv)
if ((deviceProp.sharedMemPerMultiprocessor >= SHMEM_SZ) && (selected_kernel != simple_bf16mma_gemm)) {
printf("Computing using high performance kernel = %d - %s\n", selected_kernel, kernelNames[selected_kernel]);
switch (selected_kernel)
{
case bf16mma_shmem_gemm_async_copy :
default:
checkCudaErrors(cudaFuncSetAttribute(compute_bf16gemm_async_copy, cudaFuncAttributeMaxDynamicSharedMemorySize, SHMEM_SZ));
checkKernelErrors((compute_bf16gemm_async_copy<<<deviceProp.multiProcessorCount*2, THREADS_PER_BLOCK, SHMEM_SZ>>>(A, B, C, D, alpha, beta)));
break;
case bf16mma_shmem_gemm :
checkCudaErrors(cudaFuncSetAttribute(compute_bf16gemm, cudaFuncAttributeMaxDynamicSharedMemorySize, SHMEM_SZ));
checkKernelErrors((compute_bf16gemm<<<deviceProp.multiProcessorCount*2, THREADS_PER_BLOCK, SHMEM_SZ>>>(A, B, C, D, alpha, beta)));
break;
switch (selected_kernel) {
case bf16mma_shmem_gemm_async_copy:
default:
checkCudaErrors(cudaFuncSetAttribute(
compute_bf16gemm_async_copy, cudaFuncAttributeMaxDynamicSharedMemorySize, SHMEM_SZ));
checkKernelErrors(
(compute_bf16gemm_async_copy<<<deviceProp.multiProcessorCount * 2, THREADS_PER_BLOCK, SHMEM_SZ>>>(
A, B, C, D, alpha, beta)));
break;
case bf16mma_shmem_gemm:
checkCudaErrors(
cudaFuncSetAttribute(compute_bf16gemm, cudaFuncAttributeMaxDynamicSharedMemorySize, SHMEM_SZ));
checkKernelErrors((compute_bf16gemm<<<deviceProp.multiProcessorCount * 2, THREADS_PER_BLOCK, SHMEM_SZ>>>(
A, B, C, D, alpha, beta)));
break;
}
#if CPU_DEBUG
checkCudaErrors(cudaMemcpy(result_hD, D, sizeof(float)*M_GLOBAL*N_GLOBAL, cudaMemcpyDeviceToHost));
checkCudaErrors(cudaMemcpy(result_hD, D, sizeof(float) * M_GLOBAL * N_GLOBAL, cudaMemcpyDeviceToHost));
#endif
}
else {
dim3 gridDim;
dim3 blockDim;
// blockDim.x must be a multple of warpSize
// 128x4 means we have 16 warps and a block computes a 64x64 output tile
blockDim.x = 128;
@@ -783,11 +815,7 @@ int main(int argc, char **argv)
memcpy(result_host, C_h, sizeof(float) * M_GLOBAL * N_GLOBAL);
matMultiplyOnHost(A_h, B_h, result_host,
alpha, beta,
M_GLOBAL, K_GLOBAL,
K_GLOBAL, N_GLOBAL,
M_GLOBAL, N_GLOBAL);
matMultiplyOnHost(A_h, B_h, result_host, alpha, beta, M_GLOBAL, K_GLOBAL, K_GLOBAL, N_GLOBAL, M_GLOBAL, N_GLOBAL);
for (int i = 0; i < N_GLOBAL * M_GLOBAL; i++) {
if (fabs(result_hD[i] - result_host[i]) > 0.1f) {
@@ -803,15 +831,15 @@ int main(int argc, char **argv)
checkCudaErrors(cudaEventElapsedTime(&milliseconds, start, stop));
printf("Time: %f ms\n", milliseconds);
printf("TFLOPS: %.2f\n", (((double)M_GLOBAL * N_GLOBAL * K_GLOBAL * 2)/(milliseconds/1000.)) / 1e12);
printf("TFLOPS: %.2f\n", (((double)M_GLOBAL * N_GLOBAL * K_GLOBAL * 2) / (milliseconds / 1000.)) / 1e12);
free(A_h);
free(B_h);
free(C_h);
checkCudaErrors(cudaFree((void*)A));
checkCudaErrors(cudaFree((void*)B));
checkCudaErrors(cudaFree((void*)C));
checkCudaErrors(cudaFree((void*)D));
checkCudaErrors(cudaFree((void *)A));
checkCudaErrors(cudaFree((void *)B));
checkCudaErrors(cudaFree((void *)C));
checkCudaErrors(cudaFree((void *)D));
return 0;
}

157
Samples/3_CUDA_Features/binaryPartitionCG/binaryPartitionCG.cu
View File

@@ -43,17 +43,18 @@
* divergence is inevitable one can use binary_partition group.
*/
#include <stdio.h>
#include <cooperative_groups.h>
#include <cooperative_groups/reduce.h>
#include <helper_cuda.h>
#include <stdio.h>
namespace cg = cooperative_groups;
void initOddEvenArr(int *inputArr, unsigned int size) {
for (int i = 0; i < size; i++) {
inputArr[i] = rand() % 50;
}
void initOddEvenArr(int *inputArr, unsigned int size)
{
for (int i = 0; i < size; i++) {
inputArr[i] = rand() % 50;
}
}
/**
@@ -61,99 +62,97 @@ void initOddEvenArr(int *inputArr, unsigned int size) {
*
* Creates cooperative groups and performs odd/even counting & summation.
*/
__global__ void oddEvenCountAndSumCG(int *inputArr, int *numOfOdds,
int *sumOfOddAndEvens, unsigned int size) {
cg::thread_block cta = cg::this_thread_block();
cg::grid_group grid = cg::this_grid();
cg::thread_block_tile<32> tile32 = cg::tiled_partition<32>(cta);
__global__ void oddEvenCountAndSumCG(int *inputArr, int *numOfOdds, int *sumOfOddAndEvens, unsigned int size)
{
cg::thread_block cta = cg::this_thread_block();
cg::grid_group grid = cg::this_grid();
cg::thread_block_tile<32> tile32 = cg::tiled_partition<32>(cta);
for (int i = grid.thread_rank(); i < size; i += grid.size()) {
int elem = inputArr[i];
auto subTile = cg::binary_partition(tile32, elem & 1);
if (elem & 1) // Odd numbers group
{
int oddGroupSum = cg::reduce(subTile, elem, cg::plus<int>());
for (int i = grid.thread_rank(); i < size; i += grid.size()) {
int elem = inputArr[i];
auto subTile = cg::binary_partition(tile32, elem & 1);
if (elem & 1) // Odd numbers group
{
int oddGroupSum = cg::reduce(subTile, elem, cg::plus<int>());
if (subTile.thread_rank() == 0) {
// Add number of odds present in this group of Odds.
atomicAdd(numOfOdds, subTile.size());
if (subTile.thread_rank() == 0) {
// Add number of odds present in this group of Odds.
atomicAdd(numOfOdds, subTile.size());
// Add local reduction of odds present in this group of Odds.
atomicAdd(&sumOfOddAndEvens[0], oddGroupSum);
}
} else // Even numbers group
{
int evenGroupSum = cg::reduce(subTile, elem, cg::plus<int>());
// Add local reduction of odds present in this group of Odds.
atomicAdd(&sumOfOddAndEvens[0], oddGroupSum);
}
}
else // Even numbers group
{
int evenGroupSum = cg::reduce(subTile, elem, cg::plus<int>());
if (subTile.thread_rank() == 0) {
// Add local reduction of even present in this group of evens.
atomicAdd(&sumOfOddAndEvens[1], evenGroupSum);
}
if (subTile.thread_rank() == 0) {
// Add local reduction of even present in this group of evens.
atomicAdd(&sumOfOddAndEvens[1], evenGroupSum);
}
}
// reconverge warp so for next loop iteration we ensure convergence of
// above diverged threads to perform coalesced loads of inputArr.
cg::sync(tile32);
}
// reconverge warp so for next loop iteration we ensure convergence of
// above diverged threads to perform coalesced loads of inputArr.
cg::sync(tile32);
}
}
/**
* Host main routine
*/
int main(int argc, const char **argv) {
int deviceId = findCudaDevice(argc, argv);
int *h_inputArr, *d_inputArr;
int *h_numOfOdds, *d_numOfOdds;
int *h_sumOfOddEvenElems, *d_sumOfOddEvenElems;
unsigned int arrSize = 1024 * 100;
int main(int argc, const char **argv)
{
int deviceId = findCudaDevice(argc, argv);
int *h_inputArr, *d_inputArr;
int *h_numOfOdds, *d_numOfOdds;
int *h_sumOfOddEvenElems, *d_sumOfOddEvenElems;
unsigned int arrSize = 1024 * 100;
checkCudaErrors(cudaMallocHost(&h_inputArr, sizeof(int) * arrSize));
checkCudaErrors(cudaMallocHost(&h_numOfOdds, sizeof(int)));
checkCudaErrors(cudaMallocHost(&h_sumOfOddEvenElems, sizeof(int) * 2));
initOddEvenArr(h_inputArr, arrSize);
checkCudaErrors(cudaMallocHost(&h_inputArr, sizeof(int) * arrSize));
checkCudaErrors(cudaMallocHost(&h_numOfOdds, sizeof(int)));
checkCudaErrors(cudaMallocHost(&h_sumOfOddEvenElems, sizeof(int) * 2));
initOddEvenArr(h_inputArr, arrSize);
cudaStream_t stream;
checkCudaErrors(cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking));
checkCudaErrors(cudaMalloc(&d_inputArr, sizeof(int) * arrSize));
checkCudaErrors(cudaMalloc(&d_numOfOdds, sizeof(int)));
checkCudaErrors(cudaMalloc(&d_sumOfOddEvenElems, sizeof(int) * 2));
cudaStream_t stream;
checkCudaErrors(cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking));
checkCudaErrors(cudaMalloc(&d_inputArr, sizeof(int) * arrSize));
checkCudaErrors(cudaMalloc(&d_numOfOdds, sizeof(int)));
checkCudaErrors(cudaMalloc(&d_sumOfOddEvenElems, sizeof(int) * 2));
checkCudaErrors(cudaMemcpyAsync(d_inputArr, h_inputArr, sizeof(int) * arrSize,
cudaMemcpyHostToDevice, stream));
checkCudaErrors(cudaMemsetAsync(d_numOfOdds, 0, sizeof(int), stream));
checkCudaErrors(
cudaMemsetAsync(d_sumOfOddEvenElems, 0, 2 * sizeof(int), stream));
checkCudaErrors(cudaMemcpyAsync(d_inputArr, h_inputArr, sizeof(int) * arrSize, cudaMemcpyHostToDevice, stream));
checkCudaErrors(cudaMemsetAsync(d_numOfOdds, 0, sizeof(int), stream));
checkCudaErrors(cudaMemsetAsync(d_sumOfOddEvenElems, 0, 2 * sizeof(int), stream));
// Launch the kernel
int threadsPerBlock = 0;
int blocksPerGrid = 0;
checkCudaErrors(cudaOccupancyMaxPotentialBlockSize(
&blocksPerGrid, &threadsPerBlock, oddEvenCountAndSumCG, 0, 0));
// Launch the kernel
int threadsPerBlock = 0;
int blocksPerGrid = 0;
checkCudaErrors(cudaOccupancyMaxPotentialBlockSize(&blocksPerGrid, &threadsPerBlock, oddEvenCountAndSumCG, 0, 0));
printf("\nLaunching %d blocks with %d threads...\n\n", blocksPerGrid,
threadsPerBlock);
printf("\nLaunching %d blocks with %d threads...\n\n", blocksPerGrid, threadsPerBlock);
oddEvenCountAndSumCG<<<blocksPerGrid, threadsPerBlock, 0, stream>>>(
d_inputArr, d_numOfOdds, d_sumOfOddEvenElems, arrSize);
oddEvenCountAndSumCG<<<blocksPerGrid, threadsPerBlock, 0, stream>>>(
d_inputArr, d_numOfOdds, d_sumOfOddEvenElems, arrSize);
checkCudaErrors(cudaMemcpyAsync(h_numOfOdds, d_numOfOdds, sizeof(int),
cudaMemcpyDeviceToHost, stream));
checkCudaErrors(cudaMemcpyAsync(h_sumOfOddEvenElems, d_sumOfOddEvenElems,
2 * sizeof(int), cudaMemcpyDeviceToHost,
stream));
checkCudaErrors(cudaStreamSynchronize(stream));
checkCudaErrors(cudaMemcpyAsync(h_numOfOdds, d_numOfOdds, sizeof(int), cudaMemcpyDeviceToHost, stream));
checkCudaErrors(
cudaMemcpyAsync(h_sumOfOddEvenElems, d_sumOfOddEvenElems, 2 * sizeof(int), cudaMemcpyDeviceToHost, stream));
checkCudaErrors(cudaStreamSynchronize(stream));
printf("Array size = %d Num of Odds = %d Sum of Odds = %d Sum of Evens %d\n",
arrSize, h_numOfOdds[0], h_sumOfOddEvenElems[0],
h_sumOfOddEvenElems[1]);
printf("\n...Done.\n\n");
printf("Array size = %d Num of Odds = %d Sum of Odds = %d Sum of Evens %d\n",
arrSize,
h_numOfOdds[0],
h_sumOfOddEvenElems[0],
h_sumOfOddEvenElems[1]);
printf("\n...Done.\n\n");
checkCudaErrors(cudaFreeHost(h_inputArr));
checkCudaErrors(cudaFreeHost(h_numOfOdds));
checkCudaErrors(cudaFreeHost(h_sumOfOddEvenElems));
checkCudaErrors(cudaFreeHost(h_inputArr));
checkCudaErrors(cudaFreeHost(h_numOfOdds));
checkCudaErrors(cudaFreeHost(h_sumOfOddEvenElems));
checkCudaErrors(cudaFree(d_inputArr));
checkCudaErrors(cudaFree(d_numOfOdds));
checkCudaErrors(cudaFree(d_sumOfOddEvenElems));
checkCudaErrors(cudaFree(d_inputArr));
checkCudaErrors(cudaFree(d_numOfOdds));
checkCudaErrors(cudaFree(d_sumOfOddEvenElems));
return EXIT_SUCCESS;
return EXIT_SUCCESS;
}

1
Samples/3_CUDA_Features/bindlessTexture/README.md
View File

@@ -32,4 +32,3 @@ Download and install the [CUDA Toolkit 12.5](https://developer.nvidia.com/cuda-d
Make sure the dependencies mentioned in [Dependencies]() section above are installed.
## References (for more details)

513
Samples/3_CUDA_Features/bindlessTexture/bindlessTexture.cpp
View File

@@ -37,14 +37,12 @@
Look into the bindlessTexture_kernel.cu file for most relevant code.
*/
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <math.h>
#include <vector>
#include <helper_gl.h>
#include <math.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <vector>
#if defined(__APPLE__) || defined(MACOSX)
#pragma clang diagnostic ignored "-Wdeprecated-declarations"
#include <GLUT/glut.h>
@@ -55,31 +53,31 @@
#include <GL/freeglut.h>
#endif
#include <cuda_gl_interop.h>
#include <helper_functions.h>
#include "bindlessTexture.h"
#include <helper_functions.h>
#include <cuda_gl_interop.h>
#define MAX_EPSILON_ERROR 5.0f
#define THRESHOLD 0.15f
#define THRESHOLD 0.15f
const char *sSDKsample = "CUDA bindlessTexture";
const char *imageFilenames[] = {
"flower.ppm", "person.ppm", "sponge.ppm",
"flower.ppm",
"person.ppm",
"sponge.ppm",
};
const cudaExtent atlasSize = make_cudaExtent(4, 4, 0);
const dim3 windowSize(512, 512);
const dim3 windowBlockSize(16, 16, 1);
const dim3 windowGridSize(windowSize.x / windowBlockSize.x,
windowSize.y / windowBlockSize.y);
const dim3 windowSize(512, 512);
const dim3 windowBlockSize(16, 16, 1);
const dim3 windowGridSize(windowSize.x / windowBlockSize.x, windowSize.y / windowBlockSize.y);
float lod = 0.5; // texture mip map level
float lod = 0.5; // texture mip map level
GLuint pbo; // OpenGL pixel buffer object
struct cudaGraphicsResource *cuda_pbo_resource =
NULL; // CUDA Graphics Resource (to transfer PBO)
GLuint pbo; // OpenGL pixel buffer object
struct cudaGraphicsResource *cuda_pbo_resource = NULL; // CUDA Graphics Resource (to transfer PBO)
bool animate = true;
@@ -88,330 +86,335 @@ StopWatchInterface *timer = NULL;
uint *d_output = NULL;
// Auto-Verification Code
const int frameCheckNumber = 4;
int fpsCount = 0; // FPS count for averaging
int fpsLimit = 1; // FPS limit for sampling
int g_Index = 0;
unsigned int frameCount = 0;
unsigned int g_TotalErrors = 0;
const int frameCheckNumber = 4;
int fpsCount = 0; // FPS count for averaging
int fpsLimit = 1; // FPS limit for sampling
int g_Index = 0;
unsigned int frameCount = 0;
unsigned int g_TotalErrors = 0;
int *pArgc = NULL;
int *pArgc = NULL;
char **pArgv = NULL;
extern "C" void initAtlasAndImages(const Image *images, size_t numImages,
cudaExtent atlasSize);
extern "C" void initAtlasAndImages(const Image *images, size_t numImages, cudaExtent atlasSize);
extern "C" void deinitAtlasAndImages();
extern "C" void randomizeAtlas();
extern "C" void renderAtlasImage(dim3 gridSize, dim3 blockSize, uint *d_output,
uint imageW, uint imageH, float lod);
extern "C" void renderAtlasImage(dim3 gridSize, dim3 blockSize, uint *d_output, uint imageW, uint imageH, float lod);
void computeFPS() {
frameCount++;
fpsCount++;
void computeFPS()
{
frameCount++;
fpsCount++;
if (fpsCount == fpsLimit) {
char fps[256];
float ifps = 1.f / (sdkGetAverageTimerValue(&timer) / 1000.f);
sprintf(fps, "%s: %3.1f fps", sSDKsample, ifps);
if (fpsCount == fpsLimit) {
char fps[256];
float ifps = 1.f / (sdkGetAverageTimerValue(&timer) / 1000.f);
sprintf(fps, "%s: %3.1f fps", sSDKsample, ifps);
glutSetWindowTitle(fps);
fpsCount = 0;
glutSetWindowTitle(fps);
fpsCount = 0;
fpsLimit = (int)MAX(1.0f, ifps);
sdkResetTimer(&timer);
}
fpsLimit = (int)MAX(1.0f, ifps);
sdkResetTimer(&timer);
}
}
// render image using CUDA
void render() {
// map PBO to get CUDA device pointer
checkCudaErrors(cudaGraphicsMapResources(1, &cuda_pbo_resource, 0));
size_t num_bytes;
checkCudaErrors(cudaGraphicsResourceGetMappedPointer(
(void **)&d_output, &num_bytes, cuda_pbo_resource));
void render()
{
// map PBO to get CUDA device pointer
checkCudaErrors(cudaGraphicsMapResources(1, &cuda_pbo_resource, 0));
size_t num_bytes;
checkCudaErrors(cudaGraphicsResourceGetMappedPointer((void **)&d_output, &num_bytes, cuda_pbo_resource));
// call CUDA kernel, writing results to PBO
renderAtlasImage(windowGridSize, windowBlockSize, d_output, windowSize.x,
windowSize.y, lod);
// call CUDA kernel, writing results to PBO
renderAtlasImage(windowGridSize, windowBlockSize, d_output, windowSize.x, windowSize.y, lod);
getLastCudaError("render_kernel failed");
getLastCudaError("render_kernel failed");
checkCudaErrors(cudaGraphicsUnmapResources(1, &cuda_pbo_resource, 0));
checkCudaErrors(cudaGraphicsUnmapResources(1, &cuda_pbo_resource, 0));
}
// display results using OpenGL (called by GLUT)
void display() {
sdkStartTimer(&timer);
void display()
{
sdkStartTimer(&timer);
render();
render();
// display results
glClear(GL_COLOR_BUFFER_BIT);
// display results
glClear(GL_COLOR_BUFFER_BIT);
// draw image from PBO
glDisable(GL_DEPTH_TEST);
glRasterPos2i(0, 0);
glBindBuffer(GL_PIXEL_UNPACK_BUFFER_ARB, pbo);
glDrawPixels(windowSize.x, windowSize.y, GL_RGBA, GL_UNSIGNED_BYTE, 0);
glBindBuffer(GL_PIXEL_UNPACK_BUFFER_ARB, 0);
// draw image from PBO
glDisable(GL_DEPTH_TEST);
glRasterPos2i(0, 0);
glBindBuffer(GL_PIXEL_UNPACK_BUFFER_ARB, pbo);
glDrawPixels(windowSize.x, windowSize.y, GL_RGBA, GL_UNSIGNED_BYTE, 0);
glBindBuffer(GL_PIXEL_UNPACK_BUFFER_ARB, 0);
glutSwapBuffers();
glutReportErrors();
glutSwapBuffers();
glutReportErrors();
sdkStopTimer(&timer);
computeFPS();
sdkStopTimer(&timer);
computeFPS();
}
void idle() {
if (animate) {
lod += 0.02f;
glutPostRedisplay();
}
void idle()
{
if (animate) {
lod += 0.02f;
glutPostRedisplay();
}
}
void keyboard(unsigned char key, int x, int y) {
switch (key) {
void keyboard(unsigned char key, int x, int y)
{
switch (key) {
case 27:
#if defined(__APPLE__) || defined(MACOSX)
exit(EXIT_SUCCESS);
exit(EXIT_SUCCESS);
#else
glutDestroyWindow(glutGetWindow());
return;
glutDestroyWindow(glutGetWindow());
return;
#endif
break;
break;
case '=':
case '+':
lod += 0.25f;
break;
lod += 0.25f;
break;
case '-':
lod -= 0.25f;
break;
lod -= 0.25f;
break;
case 'r':
randomizeAtlas();
break;
randomizeAtlas();
break;
case ' ':
animate = !animate;
lod = 0.0f;
break;
animate = !animate;
lod = 0.0f;
break;
default:
break;
}
break;
}
glutPostRedisplay();
glutPostRedisplay();
}
void reshape(int x, int y) {
glViewport(0, 0, x, y);
void reshape(int x, int y)
{
glViewport(0, 0, x, y);
glMatrixMode(GL_MODELVIEW);
glLoadIdentity();
glMatrixMode(GL_MODELVIEW);
glLoadIdentity();
glMatrixMode(GL_PROJECTION);
glLoadIdentity();
glOrtho(0.0, 1.0, 0.0, 1.0, 0.0, 1.0);
glMatrixMode(GL_PROJECTION);
glLoadIdentity();
glOrtho(0.0, 1.0, 0.0, 1.0, 0.0, 1.0);
}
// Global cleanup function
// Shared by both GL and non-GL code paths
void cleanup() {
sdkDeleteTimer(&timer);
void cleanup()
{
sdkDeleteTimer(&timer);
// unregister this buffer object from CUDA C
if (cuda_pbo_resource) {
cudaGraphicsUnregisterResource(cuda_pbo_resource);
glDeleteBuffers(1, &pbo);
}
// unregister this buffer object from CUDA C
if (cuda_pbo_resource) {
cudaGraphicsUnregisterResource(cuda_pbo_resource);
glDeleteBuffers(1, &pbo);
}
}
void cleanup_all() {
cleanup();
deinitAtlasAndImages();
void cleanup_all()
{
cleanup();
deinitAtlasAndImages();
}
void initGLBuffers() {
// create pixel buffer object
glGenBuffers(1, &pbo);
glBindBuffer(GL_PIXEL_UNPACK_BUFFER_ARB, pbo);
glBufferData(GL_PIXEL_UNPACK_BUFFER_ARB,
windowSize.x * windowSize.y * sizeof(GLubyte) * 4, 0,
GL_STREAM_DRAW_ARB);
glBindBuffer(GL_PIXEL_UNPACK_BUFFER_ARB, 0);
void initGLBuffers()
{
// create pixel buffer object
glGenBuffers(1, &pbo);
glBindBuffer(GL_PIXEL_UNPACK_BUFFER_ARB, pbo);
glBufferData(GL_PIXEL_UNPACK_BUFFER_ARB, windowSize.x * windowSize.y * sizeof(GLubyte) * 4, 0, GL_STREAM_DRAW_ARB);
glBindBuffer(GL_PIXEL_UNPACK_BUFFER_ARB, 0);
// register this buffer object with CUDA
checkCudaErrors(cudaGraphicsGLRegisterBuffer(
&cuda_pbo_resource, pbo, cudaGraphicsMapFlagsWriteDiscard));
// register this buffer object with CUDA
checkCudaErrors(cudaGraphicsGLRegisterBuffer(&cuda_pbo_resource, pbo, cudaGraphicsMapFlagsWriteDiscard));
}
// Load raw data from disk
uchar *loadRawFile(const char *filename, size_t size) {
FILE *fp = fopen(filename, "rb");
uchar *loadRawFile(const char *filename, size_t size)
{
FILE *fp = fopen(filename, "rb");
if (!fp) {
fprintf(stderr, "Error opening file '%s'\n", filename);
return 0;
}
uchar *data = (uchar *)malloc(size);
size_t read = fread(data, 1, size, fp);
fclose(fp);
printf("Read '%s', %zu bytes\n", filename, read);
return data;
}
void initGL(int *argc, char **argv) {
// initialize GLUT callback functions
glutInit(argc, argv);
glutInitDisplayMode(GLUT_RGB | GLUT_DOUBLE);
glutInitWindowSize(windowSize.x, windowSize.y);
glutCreateWindow(sSDKsample);
glutDisplayFunc(display);
glutKeyboardFunc(keyboard);
glutReshapeFunc(reshape);
glutIdleFunc(idle);
if (!isGLVersionSupported(2, 0) ||
!areGLExtensionsSupported("GL_ARB_pixel_buffer_object")) {
fprintf(stderr, "Required OpenGL extensions are missing.");
exit(EXIT_FAILURE);
}
}
void runAutoTest(const char *ref_file, char *exec_path) {
size_t windowBytes = windowSize.x * windowSize.y * sizeof(GLubyte) * 4;
checkCudaErrors(cudaMalloc((void **)&d_output, windowBytes));
// render the volumeData
renderAtlasImage(windowGridSize, windowBlockSize, d_output, windowSize.x,
windowSize.y, lod);
checkCudaErrors(cudaDeviceSynchronize());
getLastCudaError("render_kernel failed");
void *h_output = malloc(windowBytes);
checkCudaErrors(
cudaMemcpy(h_output, d_output, windowBytes, cudaMemcpyDeviceToHost));
sdkDumpBin(h_output, (unsigned int)windowBytes, "bindlessTexture.bin");
bool bTestResult = sdkCompareBin2BinFloat(
"bindlessTexture.bin", sdkFindFilePath(ref_file, exec_path),
windowSize.x * windowSize.y, MAX_EPSILON_ERROR, THRESHOLD, exec_path);
checkCudaErrors(cudaFree(d_output));
free(h_output);
deinitAtlasAndImages();
sdkStopTimer(&timer);
sdkDeleteTimer(&timer);
exit(bTestResult ? EXIT_SUCCESS : EXIT_FAILURE);
}
void loadImageData(const char *exe_path) {
std::vector<Image> images;
for (size_t i = 0; i < sizeof(imageFilenames) / sizeof(imageFilenames[0]);
i++) {
unsigned int imgWidth = 0;
unsigned int imgHeight = 0;
uchar *imgData = NULL;
const char *imgPath = 0;
const char *imgFilename = imageFilenames[i];
if (exe_path) {
imgPath = sdkFindFilePath(imgFilename, exe_path);
if (!fp) {
fprintf(stderr, "Error opening file '%s'\n", filename);
return 0;
}
if (imgPath == 0) {
printf("Error finding image file '%s'\n", imgFilename);
exit(EXIT_FAILURE);
uchar *data = (uchar *)malloc(size);
size_t read = fread(data, 1, size, fp);
fclose(fp);
printf("Read '%s', %zu bytes\n", filename, read);
return data;
}
void initGL(int *argc, char **argv)
{
// initialize GLUT callback functions
glutInit(argc, argv);
glutInitDisplayMode(GLUT_RGB | GLUT_DOUBLE);
glutInitWindowSize(windowSize.x, windowSize.y);
glutCreateWindow(sSDKsample);
glutDisplayFunc(display);
glutKeyboardFunc(keyboard);
glutReshapeFunc(reshape);
glutIdleFunc(idle);
if (!isGLVersionSupported(2, 0) || !areGLExtensionsSupported("GL_ARB_pixel_buffer_object")) {
fprintf(stderr, "Required OpenGL extensions are missing.");
exit(EXIT_FAILURE);
}
}
void runAutoTest(const char *ref_file, char *exec_path)
{
size_t windowBytes = windowSize.x * windowSize.y * sizeof(GLubyte) * 4;
checkCudaErrors(cudaMalloc((void **)&d_output, windowBytes));
// render the volumeData
renderAtlasImage(windowGridSize, windowBlockSize, d_output, windowSize.x, windowSize.y, lod);
checkCudaErrors(cudaDeviceSynchronize());
getLastCudaError("render_kernel failed");
void *h_output = malloc(windowBytes);
checkCudaErrors(cudaMemcpy(h_output, d_output, windowBytes, cudaMemcpyDeviceToHost));
sdkDumpBin(h_output, (unsigned int)windowBytes, "bindlessTexture.bin");
bool bTestResult = sdkCompareBin2BinFloat("bindlessTexture.bin",
sdkFindFilePath(ref_file, exec_path),
windowSize.x * windowSize.y,
MAX_EPSILON_ERROR,
THRESHOLD,
exec_path);
checkCudaErrors(cudaFree(d_output));
free(h_output);
deinitAtlasAndImages();
sdkStopTimer(&timer);
sdkDeleteTimer(&timer);
exit(bTestResult ? EXIT_SUCCESS : EXIT_FAILURE);
}
void loadImageData(const char *exe_path)
{
std::vector<Image> images;
for (size_t i = 0; i < sizeof(imageFilenames) / sizeof(imageFilenames[0]); i++) {
unsigned int imgWidth = 0;
unsigned int imgHeight = 0;
uchar *imgData = NULL;
const char *imgPath = 0;
const char *imgFilename = imageFilenames[i];
if (exe_path) {
imgPath = sdkFindFilePath(imgFilename, exe_path);
}
if (imgPath == 0) {
printf("Error finding image file '%s'\n", imgFilename);
exit(EXIT_FAILURE);
}
sdkLoadPPM4(imgPath, (unsigned char **)&imgData, &imgWidth, &imgHeight);
if (!imgData) {
printf("Error opening file '%s'\n", imgPath);
exit(EXIT_FAILURE);
}
printf("Loaded '%s', %d x %d pixels\n", imgPath, imgWidth, imgHeight);
checkHost(imgWidth > 1);
checkHost(imgHeight > 1);
Image img;
img.size = make_cudaExtent(imgWidth, imgHeight, 0);
img.h_data = imgData;
images.push_back(img);
}
sdkLoadPPM4(imgPath, (unsigned char **)&imgData, &imgWidth, &imgHeight);
if (!imgData) {
printf("Error opening file '%s'\n", imgPath);
exit(EXIT_FAILURE);
}
printf("Loaded '%s', %d x %d pixels\n", imgPath, imgWidth, imgHeight);
checkHost(imgWidth > 1);
checkHost(imgHeight > 1);
Image img;
img.size = make_cudaExtent(imgWidth, imgHeight, 0);
img.h_data = imgData;
images.push_back(img);
}
initAtlasAndImages(&images[0], images.size(), atlasSize);
initAtlasAndImages(&images[0], images.size(), atlasSize);
}
////////////////////////////////////////////////////////////////////////////////
// Program main
////////////////////////////////////////////////////////////////////////////////
int main(int argc, char **argv) {
sdkCreateTimer(&timer);
int main(int argc, char **argv)
{
sdkCreateTimer(&timer);
pArgc = &argc;
pArgv = argv;
pArgc = &argc;
pArgv = argv;
char *ref_file = NULL;
char *ref_file = NULL;
#if defined(__linux__)
setenv("DISPLAY", ":0", 0);
setenv("DISPLAY", ":0", 0);
#endif
printf("%s Starting...\n\n", sSDKsample);
printf("%s Starting...\n\n", sSDKsample);
if (checkCmdLineFlag(argc, (const char **)argv, "file")) {
fpsLimit = frameCheckNumber;
getCmdLineArgumentString(argc, (const char **)argv, "file", &ref_file);
}
if (checkCmdLineFlag(argc, (const char **)argv, "file")) {
fpsLimit = frameCheckNumber;
getCmdLineArgumentString(argc, (const char **)argv, "file", &ref_file);
}
srand(15234);
srand(15234);
// use command-line specified CUDA device, otherwise use device with highest
// Gflops/s
findCudaDevice(argc, (const char **)argv);
// use command-line specified CUDA device, otherwise use device with highest
// Gflops/s
findCudaDevice(argc, (const char **)argv);
if (!ref_file) {
initGL(&argc, argv);
if (!ref_file) {
initGL(&argc, argv);
// OpenGL buffers
initGLBuffers();
}
// OpenGL buffers
initGLBuffers();
}
if (!checkCudaCapabilities(3, 0)) {
cleanup();
if (!checkCudaCapabilities(3, 0)) {
cleanup();
exit(EXIT_WAIVED);
}
exit(EXIT_WAIVED);
}
loadImageData(argv[0]);
loadImageData(argv[0]);
if (ref_file) {
runAutoTest(ref_file, argv[0]);
}
if (ref_file) {
runAutoTest(ref_file, argv[0]);
}
printf(
"Press space to toggle animation\n"
"Press '+' and '-' to change lod level\n"
"Press 'r' to randomize virtual atlas\n");
printf("Press space to toggle animation\n"
"Press '+' and '-' to change lod level\n"
"Press 'r' to randomize virtual atlas\n");
#if defined(__APPLE__) || defined(MACOSX)
atexit(cleanup_all);
atexit(cleanup_all);
#else
glutCloseFunc(cleanup_all);
glutCloseFunc(cleanup_all);
#endif
glutMainLoop();
glutMainLoop();
}

37
Samples/3_CUDA_Features/bindlessTexture/bindlessTexture.h
View File

@@ -29,39 +29,38 @@
#define _BINDLESSTEXTURE_CU_
// includes, cuda
#include <vector_types.h>
#include <cuda_runtime.h>
#include <vector_types.h>
// CUDA utilities and system includes
#include <helper_cuda.h>
#include <vector_types.h>
typedef unsigned int uint;
typedef unsigned int uint;
typedef unsigned char uchar;
#pragma pack(push, 4)
struct Image {
void *h_data;
cudaExtent size;
cudaResourceType type;
cudaArray_t dataArray;
cudaMipmappedArray_t mipmapArray;
cudaTextureObject_t textureObject;
struct Image
{
void *h_data;
cudaExtent size;
cudaResourceType type;
cudaArray_t dataArray;
cudaMipmappedArray_t mipmapArray;
cudaTextureObject_t textureObject;
Image() { memset(this, 0, sizeof(Image)); }
Image() { memset(this, 0, sizeof(Image)); }
};
#pragma pack(pop)
inline void _checkHost(bool test, const char *condition, const char *file,
int line, const char *func) {
if (!test) {
fprintf(stderr, "HOST error at %s:%d (%s) \"%s\" \n", file, line, condition,
func);
exit(EXIT_FAILURE);
}
inline void _checkHost(bool test, const char *condition, const char *file, int line, const char *func)
{
if (!test) {
fprintf(stderr, "HOST error at %s:%d (%s) \"%s\" \n", file, line, condition, func);
exit(EXIT_FAILURE);
}
}
#define checkHost(condition) \
_checkHost(condition, #condition, __FILE__, __LINE__, __FUNCTION__)
#define checkHost(condition) _checkHost(condition, #condition, __FILE__, __LINE__, __FUNCTION__)
#endif

506
Samples/3_CUDA_Features/bindlessTexture/bindlessTexture_kernel.cu
View File

@@ -41,26 +41,24 @@
#ifndef _BINDLESSTEXTURE_KERNEL_CU_
#define _BINDLESSTEXTURE_KERNEL_CU_
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <math.h>
#include <vector>
#include <helper_cuda.h>
#include <helper_math.h>
#include <math.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <vector>
#include "bindlessTexture.h"
// set this to just see the mipmap chain of first image
//#define SHOW_MIPMAPS
// #define SHOW_MIPMAPS
// local references to resources
Image atlasImage;
Image atlasImage;
std::vector<Image> contentImages;
float highestLod = 1.0f;
float highestLod = 1.0f;
#ifndef MAX
#define MAX(a, b) ((a > b) ? a : b)
@@ -68,22 +66,21 @@ float highestLod = 1.0f;
//////////////////////////////////////////////////////////////////////////
__host__ __device__ __inline__ uint2 encodeTextureObject(
cudaTextureObject_t obj) {
return make_uint2((uint)(obj & 0xFFFFFFFF), (uint)(obj >> 32));
__host__ __device__ __inline__ uint2 encodeTextureObject(cudaTextureObject_t obj)
{
return make_uint2((uint)(obj & 0xFFFFFFFF), (uint)(obj >> 32));
}
__host__ __device__ __inline__ cudaTextureObject_t decodeTextureObject(
uint2 obj) {
return (((cudaTextureObject_t)obj.x) | ((cudaTextureObject_t)obj.y) << 32);
__host__ __device__ __inline__ cudaTextureObject_t decodeTextureObject(uint2 obj)
{
return (((cudaTextureObject_t)obj.x) | ((cudaTextureObject_t)obj.y) << 32);
}
__device__ __inline__ float4 to_float4(uchar4 vec) {
return make_float4(vec.x, vec.y, vec.z, vec.w);
}
__device__ __inline__ float4 to_float4(uchar4 vec) { return make_float4(vec.x, vec.y, vec.z, vec.w); }
__device__ __inline__ uchar4 to_uchar4(float4 vec) {
return make_uchar4((uchar)vec.x, (uchar)vec.y, (uchar)vec.z, (uchar)vec.w);
__device__ __inline__ uchar4 to_uchar4(float4 vec)
{
return make_uchar4((uchar)vec.x, (uchar)vec.y, (uchar)vec.z, (uchar)vec.w);
}
//////////////////////////////////////////////////////////////////////////
@@ -92,48 +89,46 @@ __device__ __inline__ uchar4 to_uchar4(float4 vec) {
// the atlas texture stores the 64 bit cudaTextureObjects
// we use it for "virtual" texturing
__global__ void d_render(uchar4 *d_output, uint imageW, uint imageH, float lod,
cudaTextureObject_t atlasTexture) {
uint x = blockIdx.x * blockDim.x + threadIdx.x;
uint y = blockIdx.y * blockDim.y + threadIdx.y;
__global__ void d_render(uchar4 *d_output, uint imageW, uint imageH, float lod, cudaTextureObject_t atlasTexture)
{
uint x = blockIdx.x * blockDim.x + threadIdx.x;
uint y = blockIdx.y * blockDim.y + threadIdx.y;
float u = x / (float)imageW;
float v = y / (float)imageH;
float u = x / (float)imageW;
float v = y / (float)imageH;
if ((x < imageW) && (y < imageH)) {
// read from 2D atlas texture and decode texture object
uint2 texCoded = tex2D<uint2>(atlasTexture, u, v);
cudaTextureObject_t tex = decodeTextureObject(texCoded);
if ((x < imageW) && (y < imageH)) {
// read from 2D atlas texture and decode texture object
uint2 texCoded = tex2D<uint2>(atlasTexture, u, v);
cudaTextureObject_t tex = decodeTextureObject(texCoded);
// read from cuda texture object, use template to specify what data will be
// returned. tex2DLod allows us to pass the lod (mip map level) directly.
// There is other functions with CUDA 5, e.g. tex2DGrad, that allow you
// to pass derivatives to perform automatic mipmap/anisotropic filtering.
float4 color = tex2DLod<float4>(tex, u, 1 - v, lod);
// In our sample tex is always valid, but for something like your own
// sparse texturing you would need to make sure to handle the zero case.
// read from cuda texture object, use template to specify what data will be
// returned. tex2DLod allows us to pass the lod (mip map level) directly.
// There is other functions with CUDA 5, e.g. tex2DGrad, that allow you
// to pass derivatives to perform automatic mipmap/anisotropic filtering.
float4 color = tex2DLod<float4>(tex, u, 1 - v, lod);
// In our sample tex is always valid, but for something like your own
// sparse texturing you would need to make sure to handle the zero case.
// write output color
uint i = y * imageW + x;
d_output[i] = to_uchar4(color * 255.0);
}
// write output color
uint i = y * imageW + x;
d_output[i] = to_uchar4(color * 255.0);
}
}
extern "C" void renderAtlasImage(dim3 gridSize, dim3 blockSize,
uchar4 *d_output, uint imageW, uint imageH,
float lod) {
// psuedo animate lod
lod = fmodf(lod, highestLod * 2);
lod = highestLod - fabs(lod - highestLod);
extern "C" void renderAtlasImage(dim3 gridSize, dim3 blockSize, uchar4 *d_output, uint imageW, uint imageH, float lod)
{
// psuedo animate lod
lod = fmodf(lod, highestLod * 2);
lod = highestLod - fabs(lod - highestLod);
#ifdef SHOW_MIPMAPS
lod = 0.0f;
lod = 0.0f;
#endif
d_render<<<gridSize, blockSize>>>(d_output, imageW, imageH, lod,
atlasImage.textureObject);
d_render<<<gridSize, blockSize>>>(d_output, imageW, imageH, lod, atlasImage.textureObject);
checkCudaErrors(cudaGetLastError());
checkCudaErrors(cudaGetLastError());
}
//////////////////////////////////////////////////////////////////////////
@@ -143,287 +138,278 @@ extern "C" void renderAtlasImage(dim3 gridSize, dim3 blockSize,
// global binding points anymore. We can directly pass them as function
// arguments.
__global__ void d_mipmap(cudaSurfaceObject_t mipOutput,
cudaTextureObject_t mipInput, uint imageW,
uint imageH) {
uint x = blockIdx.x * blockDim.x + threadIdx.x;
uint y = blockIdx.y * blockDim.y + threadIdx.y;
__global__ void d_mipmap(cudaSurfaceObject_t mipOutput, cudaTextureObject_t mipInput, uint imageW, uint imageH)
{
uint x = blockIdx.x * blockDim.x + threadIdx.x;
uint y = blockIdx.y * blockDim.y + threadIdx.y;
float px = 1.0 / float(imageW);
float py = 1.0 / float(imageH);
float px = 1.0 / float(imageW);
float py = 1.0 / float(imageH);
if ((x < imageW) && (y < imageH)) {
// take the average of 4 samples
if ((x < imageW) && (y < imageH)) {
// take the average of 4 samples
// we are using the normalized access to make sure non-power-of-two textures
// behave well when downsized.
float4 color = (tex2D<float4>(mipInput, (x + 0) * px, (y + 0) * py)) +
(tex2D<float4>(mipInput, (x + 1) * px, (y + 0) * py)) +
(tex2D<float4>(mipInput, (x + 1) * px, (y + 1) * py)) +
(tex2D<float4>(mipInput, (x + 0) * px, (y + 1) * py));
// we are using the normalized access to make sure non-power-of-two textures
// behave well when downsized.
float4 color = (tex2D<float4>(mipInput, (x + 0) * px, (y + 0) * py))
+ (tex2D<float4>(mipInput, (x + 1) * px, (y + 0) * py))
+ (tex2D<float4>(mipInput, (x + 1) * px, (y + 1) * py))
+ (tex2D<float4>(mipInput, (x + 0) * px, (y + 1) * py));
color /= 4.0;
color *= 255.0;
color = fminf(color, make_float4(255.0));
color /= 4.0;
color *= 255.0;
color = fminf(color, make_float4(255.0));
surf2Dwrite(to_uchar4(color), mipOutput, x * sizeof(uchar4), y);
}
surf2Dwrite(to_uchar4(color), mipOutput, x * sizeof(uchar4), y);
}
}
void generateMipMaps(cudaMipmappedArray_t mipmapArray, cudaExtent size) {
size_t width = size.width;
size_t height = size.height;
void generateMipMaps(cudaMipmappedArray_t mipmapArray, cudaExtent size)
{
size_t width = size.width;
size_t height = size.height;
#ifdef SHOW_MIPMAPS
cudaArray_t levelFirst;
checkCudaErrors(cudaGetMipmappedArrayLevel(&levelFirst, mipmapArray, 0));
cudaArray_t levelFirst;
checkCudaErrors(cudaGetMipmappedArrayLevel(&levelFirst, mipmapArray, 0));
#endif
uint level = 0;
uint level = 0;
while (width != 1 || height != 1) {
width /= 2;
width = MAX((size_t)1, width);
height /= 2;
height = MAX((size_t)1, height);
while (width != 1 || height != 1) {
width /= 2;
width = MAX((size_t)1, width);
height /= 2;
height = MAX((size_t)1, height);
cudaArray_t levelFrom;
checkCudaErrors(cudaGetMipmappedArrayLevel(&levelFrom, mipmapArray, level));
cudaArray_t levelTo;
checkCudaErrors(
cudaGetMipmappedArrayLevel(&levelTo, mipmapArray, level + 1));
cudaArray_t levelFrom;
checkCudaErrors(cudaGetMipmappedArrayLevel(&levelFrom, mipmapArray, level));
cudaArray_t levelTo;
checkCudaErrors(cudaGetMipmappedArrayLevel(&levelTo, mipmapArray, level + 1));
cudaExtent levelToSize;
checkCudaErrors(cudaArrayGetInfo(NULL, &levelToSize, NULL, levelTo));
checkHost(levelToSize.width == width);
checkHost(levelToSize.height == height);
checkHost(levelToSize.depth == 0);
cudaExtent levelToSize;
checkCudaErrors(cudaArrayGetInfo(NULL, &levelToSize, NULL, levelTo));
checkHost(levelToSize.width == width);
checkHost(levelToSize.height == height);
checkHost(levelToSize.depth == 0);
// generate texture object for reading
cudaTextureObject_t texInput;
cudaResourceDesc texRes;
memset(&texRes, 0, sizeof(cudaResourceDesc));
// generate texture object for reading
cudaTextureObject_t texInput;
cudaResourceDesc texRes;
memset(&texRes, 0, sizeof(cudaResourceDesc));
texRes.resType = cudaResourceTypeArray;
texRes.res.array.array = levelFrom;
texRes.resType = cudaResourceTypeArray;
texRes.res.array.array = levelFrom;
cudaTextureDesc texDescr;
memset(&texDescr, 0, sizeof(cudaTextureDesc));
cudaTextureDesc texDescr;
memset(&texDescr, 0, sizeof(cudaTextureDesc));
texDescr.normalizedCoords = 1;
texDescr.filterMode = cudaFilterModeLinear;
texDescr.normalizedCoords = 1;
texDescr.filterMode = cudaFilterModeLinear;
texDescr.addressMode[0] = cudaAddressModeClamp;
texDescr.addressMode[1] = cudaAddressModeClamp;
texDescr.addressMode[2] = cudaAddressModeClamp;
texDescr.addressMode[0] = cudaAddressModeClamp;
texDescr.addressMode[1] = cudaAddressModeClamp;
texDescr.addressMode[2] = cudaAddressModeClamp;
texDescr.readMode = cudaReadModeNormalizedFloat;
texDescr.readMode = cudaReadModeNormalizedFloat;
checkCudaErrors(
cudaCreateTextureObject(&texInput, &texRes, &texDescr, NULL));
checkCudaErrors(cudaCreateTextureObject(&texInput, &texRes, &texDescr, NULL));
// generate surface object for writing
// generate surface object for writing
cudaSurfaceObject_t surfOutput;
cudaResourceDesc surfRes;
memset(&surfRes, 0, sizeof(cudaResourceDesc));
surfRes.resType = cudaResourceTypeArray;
surfRes.res.array.array = levelTo;
cudaSurfaceObject_t surfOutput;
cudaResourceDesc surfRes;
memset(&surfRes, 0, sizeof(cudaResourceDesc));
surfRes.resType = cudaResourceTypeArray;
surfRes.res.array.array = levelTo;
checkCudaErrors(cudaCreateSurfaceObject(&surfOutput, &surfRes));
checkCudaErrors(cudaCreateSurfaceObject(&surfOutput, &surfRes));
// run mipmap kernel
dim3 blockSize(16, 16, 1);
dim3 gridSize(((uint)width + blockSize.x - 1) / blockSize.x,
((uint)height + blockSize.y - 1) / blockSize.y, 1);
// run mipmap kernel
dim3 blockSize(16, 16, 1);
dim3 gridSize(((uint)width + blockSize.x - 1) / blockSize.x, ((uint)height + blockSize.y - 1) / blockSize.y, 1);
d_mipmap<<<gridSize, blockSize>>>(surfOutput, texInput, (uint)width,
(uint)height);
d_mipmap<<<gridSize, blockSize>>>(surfOutput, texInput, (uint)width, (uint)height);
checkCudaErrors(cudaDeviceSynchronize());
checkCudaErrors(cudaGetLastError());
checkCudaErrors(cudaDeviceSynchronize());
checkCudaErrors(cudaGetLastError());
checkCudaErrors(cudaDestroySurfaceObject(surfOutput));
checkCudaErrors(cudaDestroySurfaceObject(surfOutput));
checkCudaErrors(cudaDestroyTextureObject(texInput));
checkCudaErrors(cudaDestroyTextureObject(texInput));
#ifdef SHOW_MIPMAPS
// we blit the current mipmap back into first level
cudaMemcpy3DParms copyParams = {0};
copyParams.dstArray = levelFirst;
copyParams.srcArray = levelTo;
copyParams.extent = make_cudaExtent(width, height, 1);
copyParams.kind = cudaMemcpyDeviceToDevice;
checkCudaErrors(cudaMemcpy3D(&copyParams));
// we blit the current mipmap back into first level
cudaMemcpy3DParms copyParams = {0};
copyParams.dstArray = levelFirst;
copyParams.srcArray = levelTo;
copyParams.extent = make_cudaExtent(width, height, 1);
copyParams.kind = cudaMemcpyDeviceToDevice;
checkCudaErrors(cudaMemcpy3D(&copyParams));
#endif
level++;
}
level++;
}
}
uint getMipMapLevels(cudaExtent size) {
size_t sz = MAX(MAX(size.width, size.height), size.depth);
uint getMipMapLevels(cudaExtent size)
{
size_t sz = MAX(MAX(size.width, size.height), size.depth);
uint levels = 0;
uint levels = 0;
while (sz) {
sz /= 2;
levels++;
}
while (sz) {
sz /= 2;
levels++;
}
return levels;
return levels;
}
//////////////////////////////////////////////////////////////////////////
// Initalization
extern "C" void randomizeAtlas() {
uint2 *h_data = (uint2 *)atlasImage.h_data;
extern "C" void randomizeAtlas()
{
uint2 *h_data = (uint2 *)atlasImage.h_data;
// assign random texture object handles to our atlas image tiles
for (size_t i = 0; i < atlasImage.size.width * atlasImage.size.height; i++) {
// assign random texture object handles to our atlas image tiles
for (size_t i = 0; i < atlasImage.size.width * atlasImage.size.height; i++) {
#ifdef SHOW_MIPMAPS
h_data[i] = encodeTextureObject(contentImages[0].textureObject);
h_data[i] = encodeTextureObject(contentImages[0].textureObject);
#else
h_data[i] = encodeTextureObject(
contentImages[rand() % contentImages.size()].textureObject);
h_data[i] = encodeTextureObject(contentImages[rand() % contentImages.size()].textureObject);
#endif
}
}
// copy data to atlas array
cudaMemcpy3DParms copyParams = {0};
copyParams.srcPtr = make_cudaPitchedPtr(
atlasImage.h_data, atlasImage.size.width * sizeof(uint2),
atlasImage.size.width, atlasImage.size.height);
copyParams.dstArray = atlasImage.dataArray;
copyParams.extent = atlasImage.size;
copyParams.extent.depth = 1;
copyParams.kind = cudaMemcpyHostToDevice;
checkCudaErrors(cudaMemcpy3D(&copyParams));
// copy data to atlas array
cudaMemcpy3DParms copyParams = {0};
copyParams.srcPtr = make_cudaPitchedPtr(
atlasImage.h_data, atlasImage.size.width * sizeof(uint2), atlasImage.size.width, atlasImage.size.height);
copyParams.dstArray = atlasImage.dataArray;
copyParams.extent = atlasImage.size;
copyParams.extent.depth = 1;
copyParams.kind = cudaMemcpyHostToDevice;
checkCudaErrors(cudaMemcpy3D(&copyParams));
};
extern "C" void deinitAtlasAndImages() {
for (size_t i = 0; i < contentImages.size(); i++) {
Image &image = contentImages[i];
extern "C" void deinitAtlasAndImages()
{
for (size_t i = 0; i < contentImages.size(); i++) {
Image &image = contentImages[i];
if (image.h_data) {
free(image.h_data);
if (image.h_data) {
free(image.h_data);
}
if (image.textureObject) {
checkCudaErrors(cudaDestroyTextureObject(image.textureObject));
}
if (image.mipmapArray) {
checkCudaErrors(cudaFreeMipmappedArray(image.mipmapArray));
}
}
if (image.textureObject) {
checkCudaErrors(cudaDestroyTextureObject(image.textureObject));
if (atlasImage.h_data) {
free(atlasImage.h_data);
}
if (image.mipmapArray) {
checkCudaErrors(cudaFreeMipmappedArray(image.mipmapArray));
if (atlasImage.textureObject) {
checkCudaErrors(cudaDestroyTextureObject(atlasImage.textureObject));
}
}
if (atlasImage.h_data) {
free(atlasImage.h_data);
}
if (atlasImage.textureObject) {
checkCudaErrors(cudaDestroyTextureObject(atlasImage.textureObject));
}
if (atlasImage.dataArray) {
checkCudaErrors(cudaFreeArray(atlasImage.dataArray));
}
if (atlasImage.dataArray) {
checkCudaErrors(cudaFreeArray(atlasImage.dataArray));
}
}
extern "C" void initAtlasAndImages(const Image *images, size_t numImages,
cudaExtent atlasSize) {
// create individual textures
contentImages.resize(numImages);
extern "C" void initAtlasAndImages(const Image *images, size_t numImages, cudaExtent atlasSize)
{
// create individual textures
contentImages.resize(numImages);
for (size_t i = 0; i < numImages; i++) {
Image &image = contentImages[i];
image.size = images[i].size;
image.size.depth = 0;
image.type = cudaResourceTypeMipmappedArray;
for (size_t i = 0; i < numImages; i++) {
Image &image = contentImages[i];
image.size = images[i].size;
image.size.depth = 0;
image.type = cudaResourceTypeMipmappedArray;
// how many mipmaps we need
uint levels = getMipMapLevels(image.size);
highestLod = MAX(highestLod, (float)levels - 1);
// how many mipmaps we need
uint levels = getMipMapLevels(image.size);
highestLod = MAX(highestLod, (float)levels - 1);
cudaChannelFormatDesc desc = cudaCreateChannelDesc<uchar4>();
checkCudaErrors(cudaMallocMipmappedArray(&image.mipmapArray, &desc,
image.size, levels));
cudaChannelFormatDesc desc = cudaCreateChannelDesc<uchar4>();
checkCudaErrors(cudaMallocMipmappedArray(&image.mipmapArray, &desc, image.size, levels));
// upload level 0
cudaArray_t level0;
checkCudaErrors(cudaGetMipmappedArrayLevel(&level0, image.mipmapArray, 0));
// upload level 0
cudaArray_t level0;
checkCudaErrors(cudaGetMipmappedArrayLevel(&level0, image.mipmapArray, 0));
cudaMemcpy3DParms copyParams = {0};
copyParams.srcPtr =
make_cudaPitchedPtr(images[i].h_data, image.size.width * sizeof(uchar4),
image.size.width, image.size.height);
copyParams.dstArray = level0;
copyParams.extent = image.size;
copyParams.extent.depth = 1;
copyParams.kind = cudaMemcpyHostToDevice;
checkCudaErrors(cudaMemcpy3D(&copyParams));
cudaMemcpy3DParms copyParams = {0};
copyParams.srcPtr = make_cudaPitchedPtr(
images[i].h_data, image.size.width * sizeof(uchar4), image.size.width, image.size.height);
copyParams.dstArray = level0;
copyParams.extent = image.size;
copyParams.extent.depth = 1;
copyParams.kind = cudaMemcpyHostToDevice;
checkCudaErrors(cudaMemcpy3D(&copyParams));
// compute rest of mipmaps based on level 0
generateMipMaps(image.mipmapArray, image.size);
// compute rest of mipmaps based on level 0
generateMipMaps(image.mipmapArray, image.size);
// generate bindless texture object
// generate bindless texture object
cudaResourceDesc resDescr;
memset(&resDescr, 0, sizeof(cudaResourceDesc));
cudaResourceDesc resDescr;
memset(&resDescr, 0, sizeof(cudaResourceDesc));
resDescr.resType = cudaResourceTypeMipmappedArray;
resDescr.res.mipmap.mipmap = image.mipmapArray;
resDescr.resType = cudaResourceTypeMipmappedArray;
resDescr.res.mipmap.mipmap = image.mipmapArray;
cudaTextureDesc texDescr;
memset(&texDescr, 0, sizeof(cudaTextureDesc));
texDescr.normalizedCoords = 1;
texDescr.filterMode = cudaFilterModeLinear;
texDescr.mipmapFilterMode = cudaFilterModeLinear;
texDescr.addressMode[0] = cudaAddressModeClamp;
texDescr.addressMode[1] = cudaAddressModeClamp;
texDescr.addressMode[2] = cudaAddressModeClamp;
texDescr.maxMipmapLevelClamp = float(levels - 1);
texDescr.readMode = cudaReadModeNormalizedFloat;
checkCudaErrors(cudaCreateTextureObject(&image.textureObject, &resDescr, &texDescr, NULL));
}
// create atlas array
cudaChannelFormatDesc channelDesc = cudaCreateChannelDesc<uint2>();
checkCudaErrors(cudaMallocArray(&atlasImage.dataArray, &channelDesc, atlasSize.width, atlasSize.height));
atlasImage.h_data = malloc(atlasSize.width * atlasSize.height * sizeof(uint2));
atlasImage.type = cudaResourceTypeArray;
atlasImage.size = atlasSize;
cudaResourceDesc texRes;
memset(&texRes, 0, sizeof(cudaResourceDesc));
texRes.resType = cudaResourceTypeArray;
texRes.res.array.array = atlasImage.dataArray;
cudaTextureDesc texDescr;
memset(&texDescr, 0, sizeof(cudaTextureDesc));
texDescr.normalizedCoords = 1;
texDescr.filterMode = cudaFilterModeLinear;
texDescr.mipmapFilterMode = cudaFilterModeLinear;
texDescr.normalizedCoords = true;
texDescr.filterMode = cudaFilterModePoint;
texDescr.addressMode[0] = cudaAddressModeClamp;
texDescr.addressMode[1] = cudaAddressModeClamp;
texDescr.addressMode[1] = cudaAddressModeClamp;
texDescr.readMode = cudaReadModeElementType;
texDescr.addressMode[0] = cudaAddressModeClamp;
texDescr.addressMode[1] = cudaAddressModeClamp;
texDescr.addressMode[2] = cudaAddressModeClamp;
checkCudaErrors(cudaCreateTextureObject(&atlasImage.textureObject, &texRes, &texDescr, NULL));
texDescr.maxMipmapLevelClamp = float(levels - 1);
texDescr.readMode = cudaReadModeNormalizedFloat;
checkCudaErrors(cudaCreateTextureObject(&image.textureObject, &resDescr,
&texDescr, NULL));
}
// create atlas array
cudaChannelFormatDesc channelDesc = cudaCreateChannelDesc<uint2>();
checkCudaErrors(cudaMallocArray(&atlasImage.dataArray, &channelDesc,
atlasSize.width, atlasSize.height));
atlasImage.h_data =
malloc(atlasSize.width * atlasSize.height * sizeof(uint2));
atlasImage.type = cudaResourceTypeArray;
atlasImage.size = atlasSize;
cudaResourceDesc texRes;
memset(&texRes, 0, sizeof(cudaResourceDesc));
texRes.resType = cudaResourceTypeArray;
texRes.res.array.array = atlasImage.dataArray;
cudaTextureDesc texDescr;
memset(&texDescr, 0, sizeof(cudaTextureDesc));
texDescr.normalizedCoords = true;
texDescr.filterMode = cudaFilterModePoint;
texDescr.addressMode[0] = cudaAddressModeClamp;
texDescr.addressMode[1] = cudaAddressModeClamp;
texDescr.addressMode[1] = cudaAddressModeClamp;
texDescr.readMode = cudaReadModeElementType;
checkCudaErrors(cudaCreateTextureObject(&atlasImage.textureObject, &texRes,
&texDescr, NULL));
randomizeAtlas();
randomizeAtlas();
}
#endif // #ifndef _SIMPLETEXTURE3D_KERNEL_CU_
#endif // #ifndef _SIMPLETEXTURE3D_KERNEL_CU_

844
Samples/3_CUDA_Features/cdpAdvancedQuicksort/cdpAdvancedQuicksort.cu
View File

@@ -40,23 +40,25 @@
// and when.
//
////////////////////////////////////////////////////////////////////////////////
#include <thrust/random.h>
#include <thrust/device_vector.h>
#include <cooperative_groups.h>
#include <thrust/device_vector.h>
#include <thrust/random.h>
namespace cg = cooperative_groups;
#include <helper_cuda.h>
#include <helper_string.h>
#include "cdpQuicksort.h"
////////////////////////////////////////////////////////////////////////////////
// Inline PTX call to return index of highest non-zero bit in a word
////////////////////////////////////////////////////////////////////////////////
static __device__ __forceinline__ unsigned int __qsflo(unsigned int word) {
unsigned int ret;
asm volatile("bfind.u32 %0, %1;" : "=r"(ret) : "r"(word));
return ret;
static __device__ __forceinline__ unsigned int __qsflo(unsigned int word)
{
unsigned int ret;
asm volatile("bfind.u32 %0, %1;" : "=r"(ret) : "r"(word));
return ret;
}
////////////////////////////////////////////////////////////////////////////////
@@ -71,25 +73,25 @@ static __device__ __forceinline__ unsigned int __qsflo(unsigned int word) {
// to complete.
//
////////////////////////////////////////////////////////////////////////////////
template <typename T>
static __device__ T *ringbufAlloc(qsortRingbuf *ringbuf) {
// Wait for there to be space in the ring buffer. We'll retry only a fixed
// number of times and then fail, to avoid an out-of-memory deadlock.
unsigned int loop = 10000;
template <typename T> static __device__ T *ringbufAlloc(qsortRingbuf *ringbuf)
{
// Wait for there to be space in the ring buffer. We'll retry only a fixed
// number of times and then fail, to avoid an out-of-memory deadlock.
unsigned int loop = 10000;
while (((ringbuf->head - ringbuf->tail) >= ringbuf->stacksize) &&
(loop-- > 0))
;
while (((ringbuf->head - ringbuf->tail) >= ringbuf->stacksize) && (loop-- > 0))
;
if (loop == 0) return NULL;
if (loop == 0)
return NULL;
// Note that the element includes a little index book-keeping, for freeing
// later.
unsigned int index = atomicAdd((unsigned int *)&ringbuf->head, 1);
T *ret = (T *)(ringbuf->stackbase) + (index & (ringbuf->stacksize - 1));
ret->index = index;
// Note that the element includes a little index book-keeping, for freeing
// later.
unsigned int index = atomicAdd((unsigned int *)&ringbuf->head, 1);
T *ret = (T *)(ringbuf->stackbase) + (index & (ringbuf->stacksize - 1));
ret->index = index;
return ret;
return ret;
}
////////////////////////////////////////////////////////////////////////////////
@@ -101,17 +103,19 @@ static __device__ T *ringbufAlloc(qsortRingbuf *ringbuf) {
// space is now available.
//
////////////////////////////////////////////////////////////////////////////////
template <typename T>
static __device__ void ringbufFree(qsortRingbuf *ringbuf, T *data) {
unsigned int index = data->index; // Non-wrapped index to free
unsigned int count = atomicAdd((unsigned int *)&(ringbuf->count), 1) + 1;
unsigned int max = atomicMax((unsigned int *)&(ringbuf->max), index + 1);
template <typename T> static __device__ void ringbufFree(qsortRingbuf *ringbuf, T *data)
{
unsigned int index = data->index; // Non-wrapped index to free
unsigned int count = atomicAdd((unsigned int *)&(ringbuf->count), 1) + 1;
unsigned int max = atomicMax((unsigned int *)&(ringbuf->max), index + 1);
// Update the tail if need be. Note we update "max" to be the new value in
// ringbuf->max
if (max < (index + 1)) max = index + 1;
// Update the tail if need be. Note we update "max" to be the new value in
// ringbuf->max
if (max < (index + 1))
max = index + 1;
if (max == count) atomicMax((unsigned int *)&(ringbuf->tail), count);
if (max == count)
atomicMax((unsigned int *)&(ringbuf->tail), count);
}
////////////////////////////////////////////////////////////////////////////////
@@ -133,201 +137,200 @@ static __device__ void ringbufFree(qsortRingbuf *ringbuf, T *data) {
// and cover the instruction overhead.
//
////////////////////////////////////////////////////////////////////////////////
__global__ void qsort_warp(unsigned *indata, unsigned *outdata,
unsigned int offset, unsigned int len,
__global__ void qsort_warp(unsigned *indata,
unsigned *outdata,
unsigned int offset,
unsigned int len,
qsortAtomicData *atomicData,
qsortRingbuf *atomicDataStack,
unsigned int source_is_indata, unsigned int depth) {
// Handle to thread block group
cg::thread_block cta = cg::this_thread_block();
// Find my data offset, based on warp ID
unsigned int thread_id = threadIdx.x + (blockIdx.x << QSORT_BLOCKSIZE_SHIFT);
// unsigned int warp_id = threadIdx.x >> 5; // Used for debug only
unsigned int lane_id = threadIdx.x & (warpSize - 1);
// Exit if I'm outside the range of sort to be done
if (thread_id >= len) return;
//
// First part of the algorithm. Each warp counts the number of elements that
// are
// greater/less than the pivot.
//
// When a warp knows its count, it updates an atomic counter.
//
// Read in the data and the pivot. Arbitrary pivot selection for now.
unsigned pivot = indata[offset + len / 2];
unsigned data = indata[offset + thread_id];
// Count how many are <= and how many are > pivot.
// If all are <= pivot then we adjust the comparison
// because otherwise the sort will move nothing and
// we'll iterate forever.
cg::coalesced_group active = cg::coalesced_threads();
unsigned int greater = (data > pivot);
unsigned int gt_mask = active.ballot(greater);
if (gt_mask == 0) {
greater = (data >= pivot);
gt_mask = active.ballot(greater); // Must re-ballot for adjusted comparator
}
unsigned int lt_mask = active.ballot(!greater);
unsigned int gt_count = __popc(gt_mask);
unsigned int lt_count = __popc(lt_mask);
// Atomically adjust the lt_ and gt_offsets by this amount. Only one thread
// need do this. Share the result using shfl
unsigned int lt_offset, gt_offset;
if (lane_id == 0) {
if (lt_count > 0)
lt_offset = atomicAdd((unsigned int *)&atomicData->lt_offset, lt_count);
if (gt_count > 0)
gt_offset =
len - (atomicAdd((unsigned int *)&atomicData->gt_offset, gt_count) +
gt_count);
}
lt_offset =
active.shfl((int)lt_offset, 0); // Everyone pulls the offsets from lane 0
gt_offset = active.shfl((int)gt_offset, 0);
// Now compute my own personal offset within this. I need to know how many
// threads with a lane ID less than mine are going to write to the same buffer
// as me. We can use popc to implement a single-operation warp scan in this
// case.
unsigned lane_mask_lt;
asm("mov.u32 %0, %%lanemask_lt;" : "=r"(lane_mask_lt));
unsigned int my_mask = greater ? gt_mask : lt_mask;
unsigned int my_offset = __popc(my_mask & lane_mask_lt);
// Move data.
my_offset += greater ? gt_offset : lt_offset;
outdata[offset + my_offset] = data;
// Count up if we're the last warp in. If so, then Kepler will launch the next
// set of sorts directly from here.
if (lane_id == 0) {
// Count "elements written". If I wrote the last one, then trigger the next
// qsorts
unsigned int mycount = lt_count + gt_count;
if (atomicAdd((unsigned int *)&atomicData->sorted_count, mycount) +
mycount ==
len) {
// We're the last warp to do any sorting. Therefore it's up to us to
// launch the next stage.
unsigned int lt_len = atomicData->lt_offset;
unsigned int gt_len = atomicData->gt_offset;
cudaStream_t lstream, rstream;
cudaStreamCreateWithFlags(&lstream, cudaStreamNonBlocking);
cudaStreamCreateWithFlags(&rstream, cudaStreamNonBlocking);
// Begin by freeing our atomicData storage. It's better for the ringbuffer
// algorithm
// if we free when we're done, rather than re-using (makes for less
// fragmentation).
ringbufFree<qsortAtomicData>(atomicDataStack, atomicData);
// Exceptional case: if "lt_len" is zero, then all values in the batch
// are equal. We are then done (may need to copy into correct buffer,
// though)
if (lt_len == 0) {
if (source_is_indata)
cudaMemcpyAsync(indata + offset, outdata + offset,
gt_len * sizeof(unsigned), cudaMemcpyDeviceToDevice,
lstream);
qsortRingbuf *atomicDataStack,
unsigned int source_is_indata,
unsigned int depth)
{
// Handle to thread block group
cg::thread_block cta = cg::this_thread_block();
// Find my data offset, based on warp ID
unsigned int thread_id = threadIdx.x + (blockIdx.x << QSORT_BLOCKSIZE_SHIFT);
// unsigned int warp_id = threadIdx.x >> 5; // Used for debug only
unsigned int lane_id = threadIdx.x & (warpSize - 1);
// Exit if I'm outside the range of sort to be done
if (thread_id >= len)
return;
}
// Start with lower half first
if (lt_len > BITONICSORT_LEN) {
// If we've exceeded maximum depth, fall through to backup
// big_bitonicsort
if (depth >= QSORT_MAXDEPTH) {
// The final bitonic stage sorts in-place in "outdata". We therefore
// re-use "indata" as the out-of-range tracking buffer. For (2^n)+1
// elements we need (2^(n+1)) bytes of oor buffer. The backup qsort
// buffer is at least this large when sizeof(QTYPE) >= 2.
big_bitonicsort<<<1, BITONICSORT_LEN, 0, lstream>>>(
outdata, source_is_indata ? indata : outdata, indata, offset,
lt_len);
} else {
// Launch another quicksort. We need to allocate more storage for the
// atomic data.
if ((atomicData = ringbufAlloc<qsortAtomicData>(atomicDataStack)) ==
NULL)
printf("Stack-allocation error. Failing left child launch.\n");
else {
atomicData->lt_offset = atomicData->gt_offset =
atomicData->sorted_count = 0;
unsigned int numblocks =
(unsigned int)(lt_len + (QSORT_BLOCKSIZE - 1)) >>
QSORT_BLOCKSIZE_SHIFT;
qsort_warp<<<numblocks, QSORT_BLOCKSIZE, 0, lstream>>>(
outdata, indata, offset, lt_len, atomicData, atomicDataStack,
!source_is_indata, depth + 1);
}
}
} else if (lt_len > 1) {
// Final stage uses a bitonic sort instead. It's important to
// make sure the final stage ends up in the correct (original) buffer.
// We launch the smallest power-of-2 number of threads that we can.
unsigned int bitonic_len = 1 << (__qsflo(lt_len - 1U) + 1);
bitonicsort<<<1, bitonic_len, 0, lstream>>>(
outdata, source_is_indata ? indata : outdata, offset, lt_len);
}
// Finally, if we sorted just one single element, we must still make
// sure that it winds up in the correct place.
else if (source_is_indata && (lt_len == 1))
indata[offset] = outdata[offset];
//
// First part of the algorithm. Each warp counts the number of elements that
// are
// greater/less than the pivot.
//
// When a warp knows its count, it updates an atomic counter.
//
if (cudaPeekAtLastError() != cudaSuccess)
printf("Left-side launch fail: %s\n",
cudaGetErrorString(cudaGetLastError()));
// Read in the data and the pivot. Arbitrary pivot selection for now.
unsigned pivot = indata[offset + len / 2];
unsigned data = indata[offset + thread_id];
// Now the upper half.
if (gt_len > BITONICSORT_LEN) {
// If we've exceeded maximum depth, fall through to backup
// big_bitonicsort
if (depth >= QSORT_MAXDEPTH)
big_bitonicsort<<<1, BITONICSORT_LEN, 0, rstream>>>(
outdata, source_is_indata ? indata : outdata, indata,
offset + lt_len, gt_len);
else {
// Allocate new atomic storage for this launch
if ((atomicData = ringbufAlloc<qsortAtomicData>(atomicDataStack)) ==
NULL)
printf("Stack allocation error! Failing right-side launch.\n");
else {
atomicData->lt_offset = atomicData->gt_offset =
atomicData->sorted_count = 0;
unsigned int numblocks =
(unsigned int)(gt_len + (QSORT_BLOCKSIZE - 1)) >>
QSORT_BLOCKSIZE_SHIFT;
qsort_warp<<<numblocks, QSORT_BLOCKSIZE, 0, rstream>>>(
outdata, indata, offset + lt_len, gt_len, atomicData,
atomicDataStack, !source_is_indata, depth + 1);
}
}
} else if (gt_len > 1) {
unsigned int bitonic_len = 1 << (__qsflo(gt_len - 1U) + 1);
bitonicsort<<<1, bitonic_len, 0, rstream>>>(
outdata, source_is_indata ? indata : outdata, offset + lt_len,
gt_len);
} else if (source_is_indata && (gt_len == 1))
indata[offset + lt_len] = outdata[offset + lt_len];
// Count how many are <= and how many are > pivot.
// If all are <= pivot then we adjust the comparison
// because otherwise the sort will move nothing and
// we'll iterate forever.
cg::coalesced_group active = cg::coalesced_threads();
unsigned int greater = (data > pivot);
unsigned int gt_mask = active.ballot(greater);
if (cudaPeekAtLastError() != cudaSuccess)
printf("Right-side launch fail: %s\n",
cudaGetErrorString(cudaGetLastError()));
if (gt_mask == 0) {
greater = (data >= pivot);
gt_mask = active.ballot(greater); // Must re-ballot for adjusted comparator
}
unsigned int lt_mask = active.ballot(!greater);
unsigned int gt_count = __popc(gt_mask);
unsigned int lt_count = __popc(lt_mask);
// Atomically adjust the lt_ and gt_offsets by this amount. Only one thread
// need do this. Share the result using shfl
unsigned int lt_offset, gt_offset;
if (lane_id == 0) {
if (lt_count > 0)
lt_offset = atomicAdd((unsigned int *)&atomicData->lt_offset, lt_count);
if (gt_count > 0)
gt_offset = len - (atomicAdd((unsigned int *)&atomicData->gt_offset, gt_count) + gt_count);
}
lt_offset = active.shfl((int)lt_offset, 0); // Everyone pulls the offsets from lane 0
gt_offset = active.shfl((int)gt_offset, 0);
// Now compute my own personal offset within this. I need to know how many
// threads with a lane ID less than mine are going to write to the same buffer
// as me. We can use popc to implement a single-operation warp scan in this
// case.
unsigned lane_mask_lt;
asm("mov.u32 %0, %%lanemask_lt;" : "=r"(lane_mask_lt));
unsigned int my_mask = greater ? gt_mask : lt_mask;
unsigned int my_offset = __popc(my_mask & lane_mask_lt);
// Move data.
my_offset += greater ? gt_offset : lt_offset;
outdata[offset + my_offset] = data;
// Count up if we're the last warp in. If so, then Kepler will launch the next
// set of sorts directly from here.
if (lane_id == 0) {
// Count "elements written". If I wrote the last one, then trigger the next
// qsorts
unsigned int mycount = lt_count + gt_count;
if (atomicAdd((unsigned int *)&atomicData->sorted_count, mycount) + mycount == len) {
// We're the last warp to do any sorting. Therefore it's up to us to
// launch the next stage.
unsigned int lt_len = atomicData->lt_offset;
unsigned int gt_len = atomicData->gt_offset;
cudaStream_t lstream, rstream;
cudaStreamCreateWithFlags(&lstream, cudaStreamNonBlocking);
cudaStreamCreateWithFlags(&rstream, cudaStreamNonBlocking);
// Begin by freeing our atomicData storage. It's better for the ringbuffer
// algorithm
// if we free when we're done, rather than re-using (makes for less
// fragmentation).
ringbufFree<qsortAtomicData>(atomicDataStack, atomicData);
// Exceptional case: if "lt_len" is zero, then all values in the batch
// are equal. We are then done (may need to copy into correct buffer,
// though)
if (lt_len == 0) {
if (source_is_indata)
cudaMemcpyAsync(indata + offset,
outdata + offset,
gt_len * sizeof(unsigned),
cudaMemcpyDeviceToDevice,
lstream);
return;
}
// Start with lower half first
if (lt_len > BITONICSORT_LEN) {
// If we've exceeded maximum depth, fall through to backup
// big_bitonicsort
if (depth >= QSORT_MAXDEPTH) {
// The final bitonic stage sorts in-place in "outdata". We therefore
// re-use "indata" as the out-of-range tracking buffer. For (2^n)+1
// elements we need (2^(n+1)) bytes of oor buffer. The backup qsort
// buffer is at least this large when sizeof(QTYPE) >= 2.
big_bitonicsort<<<1, BITONICSORT_LEN, 0, lstream>>>(
outdata, source_is_indata ? indata : outdata, indata, offset, lt_len);
}
else {
// Launch another quicksort. We need to allocate more storage for the
// atomic data.
if ((atomicData = ringbufAlloc<qsortAtomicData>(atomicDataStack)) == NULL)
printf("Stack-allocation error. Failing left child launch.\n");
else {
atomicData->lt_offset = atomicData->gt_offset = atomicData->sorted_count = 0;
unsigned int numblocks =
(unsigned int)(lt_len + (QSORT_BLOCKSIZE - 1)) >> QSORT_BLOCKSIZE_SHIFT;
qsort_warp<<<numblocks, QSORT_BLOCKSIZE, 0, lstream>>>(
outdata, indata, offset, lt_len, atomicData, atomicDataStack, !source_is_indata, depth + 1);
}
}
}
else if (lt_len > 1) {
// Final stage uses a bitonic sort instead. It's important to
// make sure the final stage ends up in the correct (original) buffer.
// We launch the smallest power-of-2 number of threads that we can.
unsigned int bitonic_len = 1 << (__qsflo(lt_len - 1U) + 1);
bitonicsort<<<1, bitonic_len, 0, lstream>>>(
outdata, source_is_indata ? indata : outdata, offset, lt_len);
}
// Finally, if we sorted just one single element, we must still make
// sure that it winds up in the correct place.
else if (source_is_indata && (lt_len == 1))
indata[offset] = outdata[offset];
if (cudaPeekAtLastError() != cudaSuccess)
printf("Left-side launch fail: %s\n", cudaGetErrorString(cudaGetLastError()));
// Now the upper half.
if (gt_len > BITONICSORT_LEN) {
// If we've exceeded maximum depth, fall through to backup
// big_bitonicsort
if (depth >= QSORT_MAXDEPTH)
big_bitonicsort<<<1, BITONICSORT_LEN, 0, rstream>>>(
outdata, source_is_indata ? indata : outdata, indata, offset + lt_len, gt_len);
else {
// Allocate new atomic storage for this launch
if ((atomicData = ringbufAlloc<qsortAtomicData>(atomicDataStack)) == NULL)
printf("Stack allocation error! Failing right-side launch.\n");
else {
atomicData->lt_offset = atomicData->gt_offset = atomicData->sorted_count = 0;
unsigned int numblocks =
(unsigned int)(gt_len + (QSORT_BLOCKSIZE - 1)) >> QSORT_BLOCKSIZE_SHIFT;
qsort_warp<<<numblocks, QSORT_BLOCKSIZE, 0, rstream>>>(outdata,
indata,
offset + lt_len,
gt_len,
atomicData,
atomicDataStack,
!source_is_indata,
depth + 1);
}
}
}
else if (gt_len > 1) {
unsigned int bitonic_len = 1 << (__qsflo(gt_len - 1U) + 1);
bitonicsort<<<1, bitonic_len, 0, rstream>>>(
outdata, source_is_indata ? indata : outdata, offset + lt_len, gt_len);
}
else if (source_is_indata && (gt_len == 1))
indata[offset + lt_len] = outdata[offset + lt_len];
if (cudaPeekAtLastError() != cudaSuccess)
printf("Right-side launch fail: %s\n", cudaGetErrorString(cudaGetLastError()));
}
}
}
}
////////////////////////////////////////////////////////////////////////////////
@@ -343,241 +346,236 @@ __global__ void qsort_warp(unsigned *indata, unsigned *outdata,
// Returns the time elapsed for the sort.
//
////////////////////////////////////////////////////////////////////////////////
float run_quicksort_cdp(unsigned *gpudata, unsigned *scratchdata,
unsigned int count, cudaStream_t stream) {
unsigned int stacksize = QSORT_STACK_ELEMS;
float run_quicksort_cdp(unsigned *gpudata, unsigned *scratchdata, unsigned int count, cudaStream_t stream)
{
unsigned int stacksize = QSORT_STACK_ELEMS;
// This is the stack, for atomic tracking of each sort's status
qsortAtomicData *gpustack;
checkCudaErrors(
cudaMalloc((void **)&gpustack, stacksize * sizeof(qsortAtomicData)));
checkCudaErrors(cudaMemset(
gpustack, 0, sizeof(qsortAtomicData))); // Only need set first entry to 0
// This is the stack, for atomic tracking of each sort's status
qsortAtomicData *gpustack;
checkCudaErrors(cudaMalloc((void **)&gpustack, stacksize * sizeof(qsortAtomicData)));
checkCudaErrors(cudaMemset(gpustack, 0, sizeof(qsortAtomicData))); // Only need set first entry to 0
// Create the memory ringbuffer used for handling the stack.
// Initialise everything to where it needs to be.
qsortRingbuf buf;
qsortRingbuf *ringbuf;
checkCudaErrors(cudaMalloc((void **)&ringbuf, sizeof(qsortRingbuf)));
buf.head = 1; // We start with one allocation
buf.tail = 0;
buf.count = 0;
buf.max = 0;
buf.stacksize = stacksize;
buf.stackbase = gpustack;
checkCudaErrors(
cudaMemcpy(ringbuf, &buf, sizeof(buf), cudaMemcpyHostToDevice));
// Create the memory ringbuffer used for handling the stack.
// Initialise everything to where it needs to be.
qsortRingbuf buf;
qsortRingbuf *ringbuf;
checkCudaErrors(cudaMalloc((void **)&ringbuf, sizeof(qsortRingbuf)));
buf.head = 1; // We start with one allocation
buf.tail = 0;
buf.count = 0;
buf.max = 0;
buf.stacksize = stacksize;
buf.stackbase = gpustack;
checkCudaErrors(cudaMemcpy(ringbuf, &buf, sizeof(buf), cudaMemcpyHostToDevice));
// Timing events...
cudaEvent_t ev1, ev2;
checkCudaErrors(cudaEventCreate(&ev1));
checkCudaErrors(cudaEventCreate(&ev2));
checkCudaErrors(cudaEventRecord(ev1));
// Timing events...
cudaEvent_t ev1, ev2;
checkCudaErrors(cudaEventCreate(&ev1));
checkCudaErrors(cudaEventCreate(&ev2));
checkCudaErrors(cudaEventRecord(ev1));
// Now we trivially launch the qsort kernel
if (count > BITONICSORT_LEN) {
unsigned int numblocks =
(unsigned int)(count + (QSORT_BLOCKSIZE - 1)) >> QSORT_BLOCKSIZE_SHIFT;
qsort_warp<<<numblocks, QSORT_BLOCKSIZE, 0, stream>>>(
gpudata, scratchdata, 0U, count, gpustack, ringbuf, true, 0);
} else {
bitonicsort<<<1, BITONICSORT_LEN>>>(gpudata, gpudata, 0, count);
}
checkCudaErrors(cudaGetLastError());
checkCudaErrors(cudaEventRecord(ev2));
checkCudaErrors(cudaDeviceSynchronize());
float elapse = 0.0f;
if (cudaPeekAtLastError() != cudaSuccess)
printf("Launch failure: %s\n", cudaGetErrorString(cudaGetLastError()));
else
checkCudaErrors(cudaEventElapsedTime(&elapse, ev1, ev2));
// Sanity check that the stack allocator is doing the right thing
checkCudaErrors(
cudaMemcpy(&buf, ringbuf, sizeof(*ringbuf), cudaMemcpyDeviceToHost));
if (count > BITONICSORT_LEN && buf.head != buf.tail) {
printf("Stack allocation error!\nRingbuf:\n");
printf("\t head = %u\n", buf.head);
printf("\t tail = %u\n", buf.tail);
printf("\tcount = %u\n", buf.count);
printf("\t max = %u\n", buf.max);
}
// Release our stack data once we're done
checkCudaErrors(cudaFree(ringbuf));
checkCudaErrors(cudaFree(gpustack));
return elapse;
}
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
int run_qsort(unsigned int size, int seed, int debug, int loop, int verbose) {
if (seed > 0) srand(seed);
// Create and set up our test
unsigned *gpudata, *scratchdata;
checkCudaErrors(cudaMalloc((void **)&gpudata, size * sizeof(unsigned)));
checkCudaErrors(cudaMalloc((void **)&scratchdata, size * sizeof(unsigned)));
// Create CPU data.
unsigned *data = new unsigned[size];
unsigned int min = loop ? loop : size;
unsigned int max = size;
loop = (loop == 0) ? 1 : loop;
for (size = min; size <= max; size += loop) {
if (verbose) printf(" Input: ");
for (unsigned int i = 0; i < size; i++) {
// Build data 8 bits at a time
data[i] = 0;
char *ptr = (char *)&(data[i]);
for (unsigned j = 0; j < sizeof(unsigned); j++) {
// Easy-to-read data in debug mode
if (debug) {
*ptr++ = (char)(rand() % 10);
break;
}
*ptr++ = (char)(rand() & 255);
}
if (verbose) {
if (i && !(i % 32)) printf("\n ");
printf("%u ", data[i]);
}
// Now we trivially launch the qsort kernel
if (count > BITONICSORT_LEN) {
unsigned int numblocks = (unsigned int)(count + (QSORT_BLOCKSIZE - 1)) >> QSORT_BLOCKSIZE_SHIFT;
qsort_warp<<<numblocks, QSORT_BLOCKSIZE, 0, stream>>>(
gpudata, scratchdata, 0U, count, gpustack, ringbuf, true, 0);
}
else {
bitonicsort<<<1, BITONICSORT_LEN>>>(gpudata, gpudata, 0, count);
}
if (verbose) printf("\n");
checkCudaErrors(cudaMemcpy(gpudata, data, size * sizeof(unsigned),
cudaMemcpyHostToDevice));
// So we're now populated and ready to go! We size our launch as
// blocks of up to BLOCKSIZE threads, and appropriate grid size.
// One thread is launched per element.
float elapse;
elapse = run_quicksort_cdp(gpudata, scratchdata, size, NULL);
// run_bitonicsort<SORTTYPE>(gpudata, scratchdata, size, verbose);
checkCudaErrors(cudaGetLastError());
checkCudaErrors(cudaEventRecord(ev2));
checkCudaErrors(cudaDeviceSynchronize());
// Copy back the data and verify correct sort
checkCudaErrors(cudaMemcpy(data, gpudata, size * sizeof(unsigned),
cudaMemcpyDeviceToHost));
float elapse = 0.0f;
if (verbose) {
printf("Output: ");
if (cudaPeekAtLastError() != cudaSuccess)
printf("Launch failure: %s\n", cudaGetErrorString(cudaGetLastError()));
else
checkCudaErrors(cudaEventElapsedTime(&elapse, ev1, ev2));
for (unsigned int i = 0; i < size; i++) {
if (i && !(i % 32)) printf("\n ");
// Sanity check that the stack allocator is doing the right thing
checkCudaErrors(cudaMemcpy(&buf, ringbuf, sizeof(*ringbuf), cudaMemcpyDeviceToHost));
printf("%u ", data[i]);
}
printf("\n");
if (count > BITONICSORT_LEN && buf.head != buf.tail) {
printf("Stack allocation error!\nRingbuf:\n");
printf("\t head = %u\n", buf.head);
printf("\t tail = %u\n", buf.tail);
printf("\tcount = %u\n", buf.count);
printf("\t max = %u\n", buf.max);
}
unsigned int check;
// Release our stack data once we're done
checkCudaErrors(cudaFree(ringbuf));
checkCudaErrors(cudaFree(gpustack));
for (check = 1; check < size; check++) {
if (data[check] < data[check - 1]) {
printf("FAILED at element: %d\n", check);
break;
}
}
if (check != size) {
printf(" cdpAdvancedQuicksort FAILED\n");
exit(EXIT_FAILURE);
} else
printf(" cdpAdvancedQuicksort PASSED\n");
// Display the time between event recordings
printf("Sorted %u elems in %.3f ms (%.3f Melems/sec)\n", size, elapse,
(float)size / (elapse * 1000.0f));
fflush(stdout);
}
// Release everything and we're done
checkCudaErrors(cudaFree(scratchdata));
checkCudaErrors(cudaFree(gpudata));
delete (data);
return 0;
return elapse;
}
static void usage() {
printf(
"Syntax: cdpAdvancedQuicksort [-size=<num>] [-seed=<num>] [-debug] "
"[-loop-step=<num>] [-verbose]\n");
printf(
"If loop_step is non-zero, will run from 1->array_len in steps of "
"loop_step\n");
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
int run_qsort(unsigned int size, int seed, int debug, int loop, int verbose)
{
if (seed > 0)
srand(seed);
// Create and set up our test
unsigned *gpudata, *scratchdata;
checkCudaErrors(cudaMalloc((void **)&gpudata, size * sizeof(unsigned)));
checkCudaErrors(cudaMalloc((void **)&scratchdata, size * sizeof(unsigned)));
// Create CPU data.
unsigned *data = new unsigned[size];
unsigned int min = loop ? loop : size;
unsigned int max = size;
loop = (loop == 0) ? 1 : loop;
for (size = min; size <= max; size += loop) {
if (verbose)
printf(" Input: ");
for (unsigned int i = 0; i < size; i++) {
// Build data 8 bits at a time
data[i] = 0;
char *ptr = (char *)&(data[i]);
for (unsigned j = 0; j < sizeof(unsigned); j++) {
// Easy-to-read data in debug mode
if (debug) {
*ptr++ = (char)(rand() % 10);
break;
}
*ptr++ = (char)(rand() & 255);
}
if (verbose) {
if (i && !(i % 32))
printf("\n ");
printf("%u ", data[i]);
}
}
if (verbose)
printf("\n");
checkCudaErrors(cudaMemcpy(gpudata, data, size * sizeof(unsigned), cudaMemcpyHostToDevice));
// So we're now populated and ready to go! We size our launch as
// blocks of up to BLOCKSIZE threads, and appropriate grid size.
// One thread is launched per element.
float elapse;
elapse = run_quicksort_cdp(gpudata, scratchdata, size, NULL);
// run_bitonicsort<SORTTYPE>(gpudata, scratchdata, size, verbose);
checkCudaErrors(cudaDeviceSynchronize());
// Copy back the data and verify correct sort
checkCudaErrors(cudaMemcpy(data, gpudata, size * sizeof(unsigned), cudaMemcpyDeviceToHost));
if (verbose) {
printf("Output: ");
for (unsigned int i = 0; i < size; i++) {
if (i && !(i % 32))
printf("\n ");
printf("%u ", data[i]);
}
printf("\n");
}
unsigned int check;
for (check = 1; check < size; check++) {
if (data[check] < data[check - 1]) {
printf("FAILED at element: %d\n", check);
break;
}
}
if (check != size) {
printf(" cdpAdvancedQuicksort FAILED\n");
exit(EXIT_FAILURE);
}
else
printf(" cdpAdvancedQuicksort PASSED\n");
// Display the time between event recordings
printf("Sorted %u elems in %.3f ms (%.3f Melems/sec)\n", size, elapse, (float)size / (elapse * 1000.0f));
fflush(stdout);
}
// Release everything and we're done
checkCudaErrors(cudaFree(scratchdata));
checkCudaErrors(cudaFree(gpudata));
delete (data);
return 0;
}
static void usage()
{
printf("Syntax: cdpAdvancedQuicksort [-size=<num>] [-seed=<num>] [-debug] "
"[-loop-step=<num>] [-verbose]\n");
printf("If loop_step is non-zero, will run from 1->array_len in steps of "
"loop_step\n");
}
// Host side entry
int main(int argc, char *argv[]) {
int size = 1000000;
unsigned int seed = 0;
int debug = 0;
int loop = 0;
int verbose = 0;
int main(int argc, char *argv[])
{
int size = 1000000;
unsigned int seed = 0;
int debug = 0;
int loop = 0;
int verbose = 0;
if (checkCmdLineFlag(argc, (const char **)argv, "help") ||
checkCmdLineFlag(argc, (const char **)argv, "h")) {
usage();
printf("&&&& cdpAdvancedQuicksort WAIVED\n");
exit(EXIT_WAIVED);
}
if (checkCmdLineFlag(argc, (const char **)argv, "help") || checkCmdLineFlag(argc, (const char **)argv, "h")) {
usage();
printf("&&&& cdpAdvancedQuicksort WAIVED\n");
exit(EXIT_WAIVED);
}
if (checkCmdLineFlag(argc, (const char **)argv, "size")) {
size = getCmdLineArgumentInt(argc, (const char **)argv, "size");
}
if (checkCmdLineFlag(argc, (const char **)argv, "size")) {
size = getCmdLineArgumentInt(argc, (const char **)argv, "size");
}
if (checkCmdLineFlag(argc, (const char **)argv, "seed")) {
seed = getCmdLineArgumentInt(argc, (const char **)argv, "seed");
}
if (checkCmdLineFlag(argc, (const char **)argv, "seed")) {
seed = getCmdLineArgumentInt(argc, (const char **)argv, "seed");
}
if (checkCmdLineFlag(argc, (const char **)argv, "loop-step")) {
loop = getCmdLineArgumentInt(argc, (const char **)argv, "loop-step");
}
if (checkCmdLineFlag(argc, (const char **)argv, "loop-step")) {
loop = getCmdLineArgumentInt(argc, (const char **)argv, "loop-step");
}
if (checkCmdLineFlag(argc, (const char **)argv, "debug")) {
debug = 1;
}
if (checkCmdLineFlag(argc, (const char **)argv, "debug")) {
debug = 1;
}
if (checkCmdLineFlag(argc, (const char **)argv, "verbose")) {
verbose = 1;
}
if (checkCmdLineFlag(argc, (const char **)argv, "verbose")) {
verbose = 1;
}
// Get device properties
int cuda_device = findCudaDevice(argc, (const char **)argv);
cudaDeviceProp properties;
checkCudaErrors(cudaGetDeviceProperties(&properties, cuda_device));
int cdpCapable =
(properties.major == 3 && properties.minor >= 5) || properties.major >= 4;
// Get device properties
int cuda_device = findCudaDevice(argc, (const char **)argv);
cudaDeviceProp properties;
checkCudaErrors(cudaGetDeviceProperties(&properties, cuda_device));
int cdpCapable = (properties.major == 3 && properties.minor >= 5) || properties.major >= 4;
printf("GPU device %s has compute capabilities (SM %d.%d)\n", properties.name,
properties.major, properties.minor);
printf("GPU device %s has compute capabilities (SM %d.%d)\n", properties.name, properties.major, properties.minor);
if (!cdpCapable) {
printf(
"cdpAdvancedQuicksort requires SM 3.5 or higher to use CUDA Dynamic "
"Parallelism. Exiting...\n");
exit(EXIT_WAIVED);
}
if (!cdpCapable) {
printf("cdpAdvancedQuicksort requires SM 3.5 or higher to use CUDA Dynamic "
"Parallelism. Exiting...\n");
exit(EXIT_WAIVED);
}
printf("Running qsort on %d elements with seed %d, on %s\n", size, seed,
properties.name);
printf("Running qsort on %d elements with seed %d, on %s\n", size, seed, properties.name);
run_qsort(size, seed, debug, loop, verbose);
run_qsort(size, seed, debug, loop, verbose);
exit(EXIT_SUCCESS);
exit(EXIT_SUCCESS);
}

328
Samples/3_CUDA_Features/cdpAdvancedQuicksort/cdpBitonicSort.cu
View File

@@ -33,18 +33,19 @@
//
// The multithread code is from me.
#include <stdio.h>
#include <cooperative_groups.h>
#include <stdio.h>
namespace cg = cooperative_groups;
#include "cdpQuicksort.h"
// Inline PTX call to return index of highest non-zero bit in a word
static __device__ __forceinline__ unsigned int __btflo(unsigned int word) {
unsigned int ret;
asm volatile("bfind.u32 %0, %1;" : "=r"(ret) : "r"(word));
return ret;
static __device__ __forceinline__ unsigned int __btflo(unsigned int word)
{
unsigned int ret;
asm volatile("bfind.u32 %0, %1;" : "=r"(ret) : "r"(word));
return ret;
}
////////////////////////////////////////////////////////////////////////////////
@@ -57,8 +58,9 @@ static __device__ __forceinline__ unsigned int __btflo(unsigned int word) {
// Perhaps it requires a class?
//
////////////////////////////////////////////////////////////////////////////////
__device__ __forceinline__ int qcompare(unsigned &val1, unsigned &val2) {
return (val1 > val2) ? 1 : (val1 == val2) ? 0 : -1;
__device__ __forceinline__ int qcompare(unsigned &val1, unsigned &val2)
{
return (val1 > val2) ? 1 : (val1 == val2) ? 0 : -1;
}
////////////////////////////////////////////////////////////////////////////////
@@ -84,68 +86,67 @@ __device__ __forceinline__ int qcompare(unsigned &val1, unsigned &val2) {
// how much data we can sort per block.
//
////////////////////////////////////////////////////////////////////////////////
static __device__ __forceinline__ void bitonicsort_kernel(
unsigned *indata, unsigned *outdata, unsigned int offset, unsigned int len,
cg::thread_block cta) {
__shared__ unsigned
sortbuf[1024]; // Max of 1024 elements - TODO: make this dynamic
static __device__ __forceinline__ void
bitonicsort_kernel(unsigned *indata, unsigned *outdata, unsigned int offset, unsigned int len, cg::thread_block cta)
{
__shared__ unsigned sortbuf[1024]; // Max of 1024 elements - TODO: make this dynamic
// First copy data into shared memory.
unsigned int inside = (threadIdx.x < len);
sortbuf[threadIdx.x] = inside ? indata[threadIdx.x + offset] : 0xffffffffu;
cg::sync(cta);
// First copy data into shared memory.
unsigned int inside = (threadIdx.x < len);
sortbuf[threadIdx.x] = inside ? indata[threadIdx.x + offset] : 0xffffffffu;
cg::sync(cta);
// Now the sort loops
// Here, "k" is the sort level (remember bitonic does a multi-level butterfly
// style sort)
// and "j" is the partner element in the butterfly.
// Two threads each work on one butterfly, because the read/write needs to
// happen
// simultaneously
for (unsigned int k = 2; k <= blockDim.x;
k *= 2) // Butterfly stride increments in powers of 2
{
for (unsigned int j = k >> 1; j > 0;
j >>= 1) // Strides also in powers of to, up to <k
// Now the sort loops
// Here, "k" is the sort level (remember bitonic does a multi-level butterfly
// style sort)
// and "j" is the partner element in the butterfly.
// Two threads each work on one butterfly, because the read/write needs to
// happen
// simultaneously
for (unsigned int k = 2; k <= blockDim.x; k *= 2) // Butterfly stride increments in powers of 2
{
unsigned int swap_idx =
threadIdx.x ^ j; // Index of element we're compare-and-swapping with
unsigned my_elem = sortbuf[threadIdx.x];
unsigned swap_elem = sortbuf[swap_idx];
for (unsigned int j = k >> 1; j > 0; j >>= 1) // Strides also in powers of to, up to <k
{
unsigned int swap_idx = threadIdx.x ^ j; // Index of element we're compare-and-swapping with
unsigned my_elem = sortbuf[threadIdx.x];
unsigned swap_elem = sortbuf[swap_idx];
cg::sync(cta);
cg::sync(cta);
// The k'th bit of my threadid (and hence my sort item ID)
// determines if we sort ascending or descending.
// However, since threads are reading from the top AND the bottom of
// the butterfly, if my ID is > swap_idx, then ascending means mine<swap.
// Finally, if either my_elem or swap_elem is out of range, then it
// ALWAYS acts like it's the largest number.
// Confusing? It saves us two writes though.
unsigned int ascend = k * (swap_idx < threadIdx.x);
unsigned int descend = k * (swap_idx > threadIdx.x);
bool swap = false;
// The k'th bit of my threadid (and hence my sort item ID)
// determines if we sort ascending or descending.
// However, since threads are reading from the top AND the bottom of
// the butterfly, if my ID is > swap_idx, then ascending means mine<swap.
// Finally, if either my_elem or swap_elem is out of range, then it
// ALWAYS acts like it's the largest number.
// Confusing? It saves us two writes though.
unsigned int ascend = k * (swap_idx < threadIdx.x);
unsigned int descend = k * (swap_idx > threadIdx.x);
bool swap = false;
if ((threadIdx.x & k) == ascend) {
if (my_elem > swap_elem) swap = true;
}
if ((threadIdx.x & k) == ascend) {
if (my_elem > swap_elem)
swap = true;
}
if ((threadIdx.x & k) == descend) {
if (my_elem < swap_elem) swap = true;
}
if ((threadIdx.x & k) == descend) {
if (my_elem < swap_elem)
swap = true;
}
// If we had to swap, then write my data to the other element's position.
// Don't forget to track out-of-range status too!
if (swap) {
sortbuf[swap_idx] = my_elem;
}
// If we had to swap, then write my data to the other element's position.
// Don't forget to track out-of-range status too!
if (swap) {
sortbuf[swap_idx] = my_elem;
}
cg::sync(cta);
cg::sync(cta);
}
}
}
// Copy the sorted data from shared memory back to the output buffer
if (threadIdx.x < len) outdata[threadIdx.x + offset] = sortbuf[threadIdx.x];
// Copy the sorted data from shared memory back to the output buffer
if (threadIdx.x < len)
outdata[threadIdx.x + offset] = sortbuf[threadIdx.x];
}
//////////////////////////////////////////////////////////////////////////////////
@@ -161,126 +162,129 @@ static __device__ __forceinline__ void bitonicsort_kernel(
// type. It must be a directly-comparable (i.e. with max value) type.
//
////////////////////////////////////////////////////////////////////////////////
static __device__ __forceinline__ void big_bitonicsort_kernel(
unsigned *indata, unsigned *outdata, unsigned *backbuf, unsigned int offset,
unsigned int len, cg::thread_block cta) {
unsigned int len2 =
1 << (__btflo(len - 1U) + 1); // Round up len to nearest power-of-2
static __device__ __forceinline__ void big_bitonicsort_kernel(unsigned *indata,
unsigned *outdata,
unsigned *backbuf,
unsigned int offset,
unsigned int len,
cg::thread_block cta)
{
unsigned int len2 = 1 << (__btflo(len - 1U) + 1); // Round up len to nearest power-of-2
if (threadIdx.x >= len2)
return; // Early out for case where more threads launched than there is
// data
if (threadIdx.x >= len2)
return; // Early out for case where more threads launched than there is
// data
// First, set up our unused values to be the max data type.
for (unsigned int i = len; i < len2; i += blockDim.x) {
unsigned int index = i + threadIdx.x;
// First, set up our unused values to be the max data type.
for (unsigned int i = len; i < len2; i += blockDim.x) {
unsigned int index = i + threadIdx.x;
if (index < len2) {
// Must split our index between two buffers
if (index < len)
indata[index + offset] = 0xffffffffu;
else
backbuf[index + offset - len] = 0xffffffffu;
}
}
cg::sync(cta);
// Now the sort loops
// Here, "k" is the sort level (remember bitonic does a multi-level butterfly
// style sort)
// and "j" is the partner element in the butterfly.
// Two threads each work on one butterfly, because the read/write needs to
// happen
// simultaneously
for (unsigned int k = 2; k <= len2;
k *= 2) // Butterfly stride increments in powers of 2
{
for (unsigned int j = k >> 1; j > 0;
j >>= 1) // Strides also in powers of to, up to <k
{
for (unsigned int i = 0; i < len2; i += blockDim.x) {
unsigned int index = threadIdx.x + i;
unsigned int swap_idx =
index ^ j; // Index of element we're compare-and-swapping with
// Only do the swap for index<swap_idx (avoids collision between other
// threads)
if (swap_idx > index) {
unsigned my_elem, swap_elem;
if (index < len)
my_elem = indata[index + offset];
else
my_elem = backbuf[index + offset - len];
if (swap_idx < len)
swap_elem = indata[swap_idx + offset];
else
swap_elem = backbuf[swap_idx + offset - len];
// The k'th bit of my index (and hence my sort item ID)
// determines if we sort ascending or descending.
// Also, if either my_elem or swap_elem is out of range, then it
// ALWAYS acts like it's the largest number.
bool swap = false;
if ((index & k) == 0) {
if (my_elem > swap_elem) swap = true;
}
if ((index & k) == k) {
if (my_elem < swap_elem) swap = true;
}
// If we had to swap, then write my data to the other element's
// position.
if (swap) {
if (swap_idx < len)
indata[swap_idx + offset] = my_elem;
if (index < len2) {
// Must split our index between two buffers
if (index < len)
indata[index + offset] = 0xffffffffu;
else
backbuf[swap_idx + offset - len] = my_elem;
backbuf[index + offset - len] = 0xffffffffu;
}
}
cg::sync(cta);
// Now the sort loops
// Here, "k" is the sort level (remember bitonic does a multi-level butterfly
// style sort)
// and "j" is the partner element in the butterfly.
// Two threads each work on one butterfly, because the read/write needs to
// happen
// simultaneously
for (unsigned int k = 2; k <= len2; k *= 2) // Butterfly stride increments in powers of 2
{
for (unsigned int j = k >> 1; j > 0; j >>= 1) // Strides also in powers of to, up to <k
{
for (unsigned int i = 0; i < len2; i += blockDim.x) {
unsigned int index = threadIdx.x + i;
unsigned int swap_idx = index ^ j; // Index of element we're compare-and-swapping with
// Only do the swap for index<swap_idx (avoids collision between other
// threads)
if (swap_idx > index) {
unsigned my_elem, swap_elem;
if (index < len)
my_elem = indata[index + offset];
else
my_elem = backbuf[index + offset - len];
if (swap_idx < len)
swap_elem = indata[swap_idx + offset];
else
swap_elem = backbuf[swap_idx + offset - len];
// The k'th bit of my index (and hence my sort item ID)
// determines if we sort ascending or descending.
// Also, if either my_elem or swap_elem is out of range, then it
// ALWAYS acts like it's the largest number.
bool swap = false;
if ((index & k) == 0) {
if (my_elem > swap_elem)
swap = true;
}
if ((index & k) == k) {
if (my_elem < swap_elem)
swap = true;
}
// If we had to swap, then write my data to the other element's
// position.
if (swap) {
if (swap_idx < len)
indata[swap_idx + offset] = my_elem;
else
backbuf[swap_idx + offset - len] = my_elem;
if (index < len)
indata[index + offset] = swap_elem;
else
backbuf[index + offset - len] = swap_elem;
}
}
}
cg::sync(cta); // Only need to sync for each "j" pass
}
}
// Copy the sorted data from the input to the output buffer, because we sort
// in-place
if (outdata != indata) {
for (unsigned int i = 0; i < len; i += blockDim.x) {
unsigned int index = i + threadIdx.x;
if (index < len)
indata[index + offset] = swap_elem;
else
backbuf[index + offset - len] = swap_elem;
}
outdata[index + offset] = indata[index + offset];
}
}
cg::sync(cta); // Only need to sync for each "j" pass
}
}
// Copy the sorted data from the input to the output buffer, because we sort
// in-place
if (outdata != indata) {
for (unsigned int i = 0; i < len; i += blockDim.x) {
unsigned int index = i + threadIdx.x;
if (index < len) outdata[index + offset] = indata[index + offset];
}
}
}
////////////////////////////////////////////////////////////////////////////////
// KERNELS
////////////////////////////////////////////////////////////////////////////////
__global__ void bitonicsort(unsigned *indata, unsigned *outdata,
unsigned int offset, unsigned int len) {
// Handle to thread block group
cg::thread_block cta = cg::this_thread_block();
bitonicsort_kernel(indata, outdata, offset, len, cta);
__global__ void bitonicsort(unsigned *indata, unsigned *outdata, unsigned int offset, unsigned int len)
{
// Handle to thread block group
cg::thread_block cta = cg::this_thread_block();
bitonicsort_kernel(indata, outdata, offset, len, cta);
}
__global__ void big_bitonicsort(unsigned *indata, unsigned *outdata,
unsigned *backbuf, unsigned int offset,
unsigned int len) {
// Handle to thread block group
cg::thread_block cta = cg::this_thread_block();
big_bitonicsort_kernel(indata, outdata, backbuf, offset, len, cta);
__global__ void
big_bitonicsort(unsigned *indata, unsigned *outdata, unsigned *backbuf, unsigned int offset, unsigned int len)
{
// Handle to thread block group
cg::thread_block cta = cg::this_thread_block();
big_bitonicsort_kernel(indata, outdata, backbuf, offset, len, cta);
}
////////////////////////////////////////////////////////////////////////////////

64
Samples/3_CUDA_Features/cdpAdvancedQuicksort/cdpQuicksort.h
View File

@@ -31,49 +31,47 @@
#define QUICKSORT_H
#define QSORT_BLOCKSIZE_SHIFT 9
#define QSORT_BLOCKSIZE (1 << QSORT_BLOCKSIZE_SHIFT)
#define BITONICSORT_LEN 1024 // Must be power of 2!
#define QSORT_MAXDEPTH \
16 // Will force final bitonic stage at depth QSORT_MAXDEPTH+1
#define QSORT_BLOCKSIZE (1 << QSORT_BLOCKSIZE_SHIFT)
#define BITONICSORT_LEN 1024 // Must be power of 2!
#define QSORT_MAXDEPTH 16 // Will force final bitonic stage at depth QSORT_MAXDEPTH+1
////////////////////////////////////////////////////////////////////////////////
// The algorithm uses several variables updated by using atomic operations.
////////////////////////////////////////////////////////////////////////////////
typedef struct __align__(128) qsortAtomicData_t {
volatile unsigned int lt_offset; // Current output offset for <pivot
volatile unsigned int gt_offset; // Current output offset for >pivot
volatile unsigned int sorted_count; // Total count sorted, for deciding when
// to launch next wave
volatile unsigned int
index; // Ringbuf tracking index. Can be ignored if not using ringbuf.
}
qsortAtomicData;
typedef struct __align__(128) qsortAtomicData_t
{
volatile unsigned int lt_offset; // Current output offset for <pivot
volatile unsigned int gt_offset; // Current output offset for >pivot
volatile unsigned int sorted_count; // Total count sorted, for deciding when
// to launch next wave
volatile unsigned int index; // Ringbuf tracking index. Can be ignored if not using ringbuf.
} qsortAtomicData;
////////////////////////////////////////////////////////////////////////////////
// A ring-buffer for rapid stack allocation
////////////////////////////////////////////////////////////////////////////////
typedef struct qsortRingbuf_t {
volatile unsigned int head; // Head pointer - we allocate from here
volatile unsigned int
tail; // Tail pointer - indicates last still-in-use element
volatile unsigned int count; // Total count allocated
volatile unsigned int max; // Max index allocated
unsigned int stacksize; // Wrap-around size of buffer (must be power of 2)
volatile void *stackbase; // Pointer to the stack we're allocating from
typedef struct qsortRingbuf_t
{
volatile unsigned int head; // Head pointer - we allocate from here
volatile unsigned int tail; // Tail pointer - indicates last still-in-use element
volatile unsigned int count; // Total count allocated
volatile unsigned int max; // Max index allocated
unsigned int stacksize; // Wrap-around size of buffer (must be power of 2)
volatile void *stackbase; // Pointer to the stack we're allocating from
} qsortRingbuf;
// Stack elem count must be power of 2!
#define QSORT_STACK_ELEMS \
1 * 1024 * 1024 // One million stack elements is a HUGE number.
#define QSORT_STACK_ELEMS 1 * 1024 * 1024 // One million stack elements is a HUGE number.
__global__ void qsort_warp(unsigned *indata, unsigned *outdata,
unsigned int len, qsortAtomicData *atomicData,
qsortRingbuf *ringbuf, unsigned int source_is_indata,
unsigned int depth);
__global__ void bitonicsort(unsigned *indata, unsigned *outdata,
unsigned int offset, unsigned int len);
__global__ void big_bitonicsort(unsigned *indata, unsigned *outdata,
unsigned *backbuf, unsigned int offset,
unsigned int len);
__global__ void qsort_warp(unsigned *indata,
unsigned *outdata,
unsigned int len,
qsortAtomicData *atomicData,
qsortRingbuf *ringbuf,
unsigned int source_is_indata,
unsigned int depth);
__global__ void bitonicsort(unsigned *indata, unsigned *outdata, unsigned int offset, unsigned int len);
__global__ void
big_bitonicsort(unsigned *indata, unsigned *outdata, unsigned *backbuf, unsigned int offset, unsigned int len);
#endif // QUICKSORT_H
#endif // QUICKSORT_H

264
Samples/3_CUDA_Features/cdpBezierTessellation/BezierLineCDP.cu
View File

@@ -25,183 +25,181 @@
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include <stdio.h>
#include <cuda_runtime_api.h>
#include <helper_cuda.h>
#include <stdio.h>
#include <string.h>
__forceinline__ __device__ float2 operator+(float2 a, float2 b) {
float2 c;
c.x = a.x + b.x;
c.y = a.y + b.y;
return c;
__forceinline__ __device__ float2 operator+(float2 a, float2 b)
{
float2 c;
c.x = a.x + b.x;
c.y = a.y + b.y;
return c;
}
__forceinline__ __device__ float2 operator-(float2 a, float2 b) {
float2 c;
c.x = a.x - b.x;
c.y = a.y - b.y;
return c;
__forceinline__ __device__ float2 operator-(float2 a, float2 b)
{
float2 c;
c.x = a.x - b.x;
c.y = a.y - b.y;
return c;
}
__forceinline__ __device__ float2 operator*(float a, float2 b) {
float2 c;
c.x = a * b.x;
c.y = a * b.y;
return c;
__forceinline__ __device__ float2 operator*(float a, float2 b)
{
float2 c;
c.x = a * b.x;
c.y = a * b.y;
return c;
}
__forceinline__ __device__ float length(float2 a) {
return sqrtf(a.x * a.x + a.y * a.y);
}
__forceinline__ __device__ float length(float2 a) { return sqrtf(a.x * a.x + a.y * a.y); }
#define MAX_TESSELLATION 32
struct BezierLine {
float2 CP[3];
float2 *vertexPos;
int nVertices;
struct BezierLine
{
float2 CP[3];
float2 *vertexPos;
int nVertices;
};
__global__ void computeBezierLinePositions(int lidx, BezierLine *bLines,
int nTessPoints) {
int idx = threadIdx.x + blockDim.x * blockIdx.x;
__global__ void computeBezierLinePositions(int lidx, BezierLine *bLines, int nTessPoints)
{
int idx = threadIdx.x + blockDim.x * blockIdx.x;
if (idx < nTessPoints) {
float u = (float)idx / (float)(nTessPoints - 1);
float omu = 1.0f - u;
if (idx < nTessPoints) {
float u = (float)idx / (float)(nTessPoints - 1);
float omu = 1.0f - u;
float B3u[3];
float B3u[3];
B3u[0] = omu * omu;
B3u[1] = 2.0f * u * omu;
B3u[2] = u * u;
B3u[0] = omu * omu;
B3u[1] = 2.0f * u * omu;
B3u[2] = u * u;
float2 position = {0, 0};
float2 position = {0, 0};
for (int i = 0; i < 3; i++) {
position = position + B3u[i] * bLines[lidx].CP[i];
for (int i = 0; i < 3; i++) {
position = position + B3u[i] * bLines[lidx].CP[i];
}
bLines[lidx].vertexPos[idx] = position;
}
bLines[lidx].vertexPos[idx] = position;
}
}
__global__ void computeBezierLinesCDP(BezierLine *bLines, int nLines) {
int lidx = threadIdx.x + blockDim.x * blockIdx.x;
__global__ void computeBezierLinesCDP(BezierLine *bLines, int nLines)
{
int lidx = threadIdx.x + blockDim.x * blockIdx.x;
if (lidx < nLines) {
float curvature = length(bLines[lidx].CP[1] -
0.5f * (bLines[lidx].CP[0] + bLines[lidx].CP[2])) /
length(bLines[lidx].CP[2] - bLines[lidx].CP[0]);
int nTessPoints = min(max((int)(curvature * 16.0f), 4), MAX_TESSELLATION);
if (lidx < nLines) {
float curvature = length(bLines[lidx].CP[1] - 0.5f * (bLines[lidx].CP[0] + bLines[lidx].CP[2]))
/ length(bLines[lidx].CP[2] - bLines[lidx].CP[0]);
int nTessPoints = min(max((int)(curvature * 16.0f), 4), MAX_TESSELLATION);
if (bLines[lidx].vertexPos == NULL) {
bLines[lidx].nVertices = nTessPoints;
cudaMalloc((void **)&bLines[lidx].vertexPos,
nTessPoints * sizeof(float2));
if (bLines[lidx].vertexPos == NULL) {
bLines[lidx].nVertices = nTessPoints;
cudaMalloc((void **)&bLines[lidx].vertexPos, nTessPoints * sizeof(float2));
}
computeBezierLinePositions<<<ceilf((float)bLines[lidx].nVertices / 32.0f), 32>>>(
lidx, bLines, bLines[lidx].nVertices);
}
computeBezierLinePositions<<<ceilf((float)bLines[lidx].nVertices / 32.0f),
32>>>(lidx, bLines, bLines[lidx].nVertices);
}
}
__global__ void freeVertexMem(BezierLine *bLines, int nLines) {
int lidx = threadIdx.x + blockDim.x * blockIdx.x;
__global__ void freeVertexMem(BezierLine *bLines, int nLines)
{
int lidx = threadIdx.x + blockDim.x * blockIdx.x;
if (lidx < nLines) cudaFree(bLines[lidx].vertexPos);
if (lidx < nLines)
cudaFree(bLines[lidx].vertexPos);
}
unsigned int checkCapableSM35Device(int argc, char **argv) {
// Get device properties
cudaDeviceProp properties;
int device_count = 0, device = -1;
if (checkCmdLineFlag(argc, (const char **)argv, "device")) {
device = getCmdLineArgumentInt(argc, (const char **)argv, "device");
unsigned int checkCapableSM35Device(int argc, char **argv)
{
// Get device properties
cudaDeviceProp properties;
checkCudaErrors(cudaGetDeviceProperties(&properties, device));
int device_count = 0, device = -1;
if (properties.major > 3 ||
(properties.major == 3 && properties.minor >= 5)) {
printf("Running on GPU %d (%s)\n", device, properties.name);
} else {
printf(
"cdpBezierTessellation requires GPU devices with compute SM 3.5 or "
"higher.");
printf("Current GPU device has compute SM %d.%d. Exiting...\n",
properties.major, properties.minor);
return EXIT_FAILURE;
if (checkCmdLineFlag(argc, (const char **)argv, "device")) {
device = getCmdLineArgumentInt(argc, (const char **)argv, "device");
cudaDeviceProp properties;
checkCudaErrors(cudaGetDeviceProperties(&properties, device));
if (properties.major > 3 || (properties.major == 3 && properties.minor >= 5)) {
printf("Running on GPU %d (%s)\n", device, properties.name);
}
else {
printf("cdpBezierTessellation requires GPU devices with compute SM 3.5 or "
"higher.");
printf("Current GPU device has compute SM %d.%d. Exiting...\n", properties.major, properties.minor);
return EXIT_FAILURE;
}
}
else {
checkCudaErrors(cudaGetDeviceCount(&device_count));
for (int i = 0; i < device_count; ++i) {
checkCudaErrors(cudaGetDeviceProperties(&properties, i));
if (properties.major > 3 || (properties.major == 3 && properties.minor >= 5)) {
device = i;
printf("Running on GPU %d (%s)\n", i, properties.name);
break;
}
printf("GPU %d %s does not support CUDA Dynamic Parallelism\n", i, properties.name);
}
}
if (device == -1) {
fprintf(stderr,
"cdpBezierTessellation requires GPU devices with compute SM 3.5 or "
"higher. Exiting...\n");
return EXIT_WAIVED;
}
} else {
checkCudaErrors(cudaGetDeviceCount(&device_count));
for (int i = 0; i < device_count; ++i) {
checkCudaErrors(cudaGetDeviceProperties(&properties, i));
if (properties.major > 3 ||
(properties.major == 3 && properties.minor >= 5)) {
device = i;
printf("Running on GPU %d (%s)\n", i, properties.name);
break;
}
printf("GPU %d %s does not support CUDA Dynamic Parallelism\n", i,
properties.name);
}
}
if (device == -1) {
fprintf(stderr,
"cdpBezierTessellation requires GPU devices with compute SM 3.5 or "
"higher. Exiting...\n");
return EXIT_WAIVED;
}
return EXIT_SUCCESS;
return EXIT_SUCCESS;
}
#define N_LINES 256
#define N_LINES 256
#define BLOCK_DIM 64
int main(int argc, char **argv) {
BezierLine *bLines_h = new BezierLine[N_LINES];
int main(int argc, char **argv)
{
BezierLine *bLines_h = new BezierLine[N_LINES];
float2 last = {0, 0};
float2 last = {0, 0};
for (int i = 0; i < N_LINES; i++) {
bLines_h[i].CP[0] = last;
for (int i = 0; i < N_LINES; i++) {
bLines_h[i].CP[0] = last;
for (int j = 1; j < 3; j++) {
bLines_h[i].CP[j].x = (float)rand() / (float)RAND_MAX;
bLines_h[i].CP[j].y = (float)rand() / (float)RAND_MAX;
for (int j = 1; j < 3; j++) {
bLines_h[i].CP[j].x = (float)rand() / (float)RAND_MAX;
bLines_h[i].CP[j].y = (float)rand() / (float)RAND_MAX;
}
last = bLines_h[i].CP[2];
bLines_h[i].vertexPos = NULL;
bLines_h[i].nVertices = 0;
}
last = bLines_h[i].CP[2];
bLines_h[i].vertexPos = NULL;
bLines_h[i].nVertices = 0;
}
unsigned int sm35Ret = checkCapableSM35Device(argc, argv);
if (sm35Ret != EXIT_SUCCESS) {
exit(sm35Ret);
}
unsigned int sm35Ret = checkCapableSM35Device(argc, argv);
if (sm35Ret != EXIT_SUCCESS) {
exit(sm35Ret);
}
BezierLine *bLines_d;
checkCudaErrors(cudaMalloc((void **)&bLines_d, N_LINES * sizeof(BezierLine)));
checkCudaErrors(cudaMemcpy(bLines_d, bLines_h, N_LINES * sizeof(BezierLine), cudaMemcpyHostToDevice));
printf("Computing Bezier Lines (CUDA Dynamic Parallelism Version) ... ");
computeBezierLinesCDP<<<(unsigned int)ceil((float)N_LINES / (float)BLOCK_DIM), BLOCK_DIM>>>(bLines_d, N_LINES);
printf("Done!\n");
BezierLine *bLines_d;
checkCudaErrors(cudaMalloc((void **)&bLines_d, N_LINES * sizeof(BezierLine)));
checkCudaErrors(cudaMemcpy(bLines_d, bLines_h, N_LINES * sizeof(BezierLine),
cudaMemcpyHostToDevice));
printf("Computing Bezier Lines (CUDA Dynamic Parallelism Version) ... ");
computeBezierLinesCDP<<<(unsigned int)ceil((float)N_LINES / (float)BLOCK_DIM),
BLOCK_DIM>>>(bLines_d, N_LINES);
printf("Done!\n");
// Do something to draw the lines here
// Do something to draw the lines here
freeVertexMem<<<(unsigned int)ceil((float)N_LINES / (float)BLOCK_DIM), BLOCK_DIM>>>(bLines_d, N_LINES);
checkCudaErrors(cudaFree(bLines_d));
delete[] bLines_h;
freeVertexMem<<<(unsigned int)ceil((float)N_LINES / (float)BLOCK_DIM),
BLOCK_DIM>>>(bLines_d, N_LINES);
checkCudaErrors(cudaFree(bLines_d));
delete[] bLines_h;
exit(EXIT_SUCCESS);
exit(EXIT_SUCCESS);
}

1027
Samples/3_CUDA_Features/cdpQuadtree/cdpQuadtree.cu
View File

File diff suppressed because it is too large Load Diff

182
Samples/3_CUDA_Features/cdpSimplePrint/cdpSimplePrint.cu
View File

@@ -25,11 +25,10 @@
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include <helper_cuda.h>
#include <helper_string.h>
#include <cstdio>
#include <cstdlib>
#include <helper_cuda.h>
#include <helper_string.h>
#include <iostream>
////////////////////////////////////////////////////////////////////////////////
@@ -40,26 +39,26 @@ __device__ int g_uids = 0;
////////////////////////////////////////////////////////////////////////////////
// Print a simple message to signal the block which is currently executing.
////////////////////////////////////////////////////////////////////////////////
__device__ void print_info(int depth, int thread, int uid, int parent_uid) {
if (threadIdx.x == 0) {
if (depth == 0)
printf("BLOCK %d launched by the host\n", uid);
else {
char buffer[32];
__device__ void print_info(int depth, int thread, int uid, int parent_uid)
{
if (threadIdx.x == 0) {
if (depth == 0)
printf("BLOCK %d launched by the host\n", uid);
else {
char buffer[32];
for (int i = 0; i < depth; ++i) {
buffer[3 * i + 0] = '|';
buffer[3 * i + 1] = ' ';
buffer[3 * i + 2] = ' ';
}
for (int i = 0; i < depth; ++i) {
buffer[3 * i + 0] = '|';
buffer[3 * i + 1] = ' ';
buffer[3 * i + 2] = ' ';
}
buffer[3 * depth] = '\0';
printf("%sBLOCK %d launched by thread %d of block %d\n", buffer, uid,
thread, parent_uid);
buffer[3 * depth] = '\0';
printf("%sBLOCK %d launched by thread %d of block %d\n", buffer, uid, thread, parent_uid);
}
}
}
__syncthreads();
__syncthreads();
}
////////////////////////////////////////////////////////////////////////////////
@@ -69,101 +68,94 @@ __device__ void print_info(int depth, int thread, int uid, int parent_uid) {
// about that block. Finally, if the 'max_depth' has not been reached, the
// block launches new blocks directly from the GPU.
////////////////////////////////////////////////////////////////////////////////
__global__ void cdp_kernel(int max_depth, int depth, int thread,
int parent_uid) {
// We create a unique ID per block. Thread 0 does that and shares the value
// with the other threads.
__shared__ int s_uid;
__global__ void cdp_kernel(int max_depth, int depth, int thread, int parent_uid)
{
// We create a unique ID per block. Thread 0 does that and shares the value
// with the other threads.
__shared__ int s_uid;
if (threadIdx.x == 0) {
s_uid = atomicAdd(&g_uids, 1);
}
if (threadIdx.x == 0) {
s_uid = atomicAdd(&g_uids, 1);
}
__syncthreads();
__syncthreads();
// We print the ID of the block and information about its parent.
print_info(depth, thread, s_uid, parent_uid);
// We print the ID of the block and information about its parent.
print_info(depth, thread, s_uid, parent_uid);
// We launch new blocks if we haven't reached the max_depth yet.
if (++depth >= max_depth) {
return;
}
// We launch new blocks if we haven't reached the max_depth yet.
if (++depth >= max_depth) {
return;
}
cdp_kernel<<<gridDim.x, blockDim.x>>>(max_depth, depth, threadIdx.x, s_uid);
cdp_kernel<<<gridDim.x, blockDim.x>>>(max_depth, depth, threadIdx.x, s_uid);
}
////////////////////////////////////////////////////////////////////////////////
// Main entry point.
////////////////////////////////////////////////////////////////////////////////
int main(int argc, char **argv) {
printf("starting Simple Print (CUDA Dynamic Parallelism)\n");
int main(int argc, char **argv)
{
printf("starting Simple Print (CUDA Dynamic Parallelism)\n");
// Parse a few command-line arguments.
int max_depth = 2;
// Parse a few command-line arguments.
int max_depth = 2;
if (checkCmdLineFlag(argc, (const char **)argv, "help") ||
checkCmdLineFlag(argc, (const char **)argv, "h")) {
printf(
"Usage: %s depth=<max_depth>\t(where max_depth is a value between 1 "
"and 8).\n",
argv[0]);
exit(EXIT_SUCCESS);
}
if (checkCmdLineFlag(argc, (const char **)argv, "depth")) {
max_depth = getCmdLineArgumentInt(argc, (const char **)argv, "depth");
if (max_depth < 1 || max_depth > 8) {
printf("depth parameter has to be between 1 and 8\n");
exit(EXIT_FAILURE);
if (checkCmdLineFlag(argc, (const char **)argv, "help") || checkCmdLineFlag(argc, (const char **)argv, "h")) {
printf("Usage: %s depth=<max_depth>\t(where max_depth is a value between 1 "
"and 8).\n",
argv[0]);
exit(EXIT_SUCCESS);
}
}
// Find/set the device.
int device = -1;
cudaDeviceProp deviceProp;
device = findCudaDevice(argc, (const char **)argv);
checkCudaErrors(cudaGetDeviceProperties(&deviceProp, device));
if (checkCmdLineFlag(argc, (const char **)argv, "depth")) {
max_depth = getCmdLineArgumentInt(argc, (const char **)argv, "depth");
if (!(deviceProp.major > 3 ||
(deviceProp.major == 3 && deviceProp.minor >= 5))) {
printf("GPU %d - %s does not support CUDA Dynamic Parallelism\n Exiting.",
device, deviceProp.name);
exit(EXIT_WAIVED);
}
if (max_depth < 1 || max_depth > 8) {
printf("depth parameter has to be between 1 and 8\n");
exit(EXIT_FAILURE);
}
}
// Print a message describing what the sample does.
printf(
"*********************************************************************"
"******\n");
printf(
"The CPU launches 2 blocks of 2 threads each. On the device each thread "
"will\n");
printf(
"launch 2 blocks of 2 threads each. The GPU we will do that "
"recursively\n");
printf("until it reaches max_depth=%d\n\n", max_depth);
printf("In total 2");
int num_blocks = 2, sum = 2;
// Find/set the device.
int device = -1;
cudaDeviceProp deviceProp;
device = findCudaDevice(argc, (const char **)argv);
checkCudaErrors(cudaGetDeviceProperties(&deviceProp, device));
for (int i = 1; i < max_depth; ++i) {
num_blocks *= 4;
printf("+%d", num_blocks);
sum += num_blocks;
}
if (!(deviceProp.major > 3 || (deviceProp.major == 3 && deviceProp.minor >= 5))) {
printf("GPU %d - %s does not support CUDA Dynamic Parallelism\n Exiting.", device, deviceProp.name);
exit(EXIT_WAIVED);
}
printf("=%d blocks are launched!!! (%d from the GPU)\n", sum, sum - 2);
printf(
"************************************************************************"
"***\n\n");
// Print a message describing what the sample does.
printf("*********************************************************************"
"******\n");
printf("The CPU launches 2 blocks of 2 threads each. On the device each thread "
"will\n");
printf("launch 2 blocks of 2 threads each. The GPU we will do that "
"recursively\n");
printf("until it reaches max_depth=%d\n\n", max_depth);
printf("In total 2");
int num_blocks = 2, sum = 2;
// Launch the kernel from the CPU.
printf("Launching cdp_kernel() with CUDA Dynamic Parallelism:\n\n");
cdp_kernel<<<2, 2>>>(max_depth, 0, 0, -1);
checkCudaErrors(cudaGetLastError());
for (int i = 1; i < max_depth; ++i) {
num_blocks *= 4;
printf("+%d", num_blocks);
sum += num_blocks;
}
// Finalize.
checkCudaErrors(cudaDeviceSynchronize());
printf("=%d blocks are launched!!! (%d from the GPU)\n", sum, sum - 2);
printf("************************************************************************"
"***\n\n");
exit(EXIT_SUCCESS);
// Launch the kernel from the CPU.
printf("Launching cdp_kernel() with CUDA Dynamic Parallelism:\n\n");
cdp_kernel<<<2, 2>>>(max_depth, 0, 0, -1);
checkCudaErrors(cudaGetLastError());
// Finalize.
checkCudaErrors(cudaDeviceSynchronize());
exit(EXIT_SUCCESS);
}

314
Samples/3_CUDA_Features/cdpSimpleQuicksort/cdpSimpleQuicksort.cu
View File

@@ -25,221 +25,221 @@
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include <iostream>
#include <cstdio>
#include <helper_cuda.h>
#include <helper_string.h>
#include <iostream>
#define MAX_DEPTH 16
#define MAX_DEPTH 16
#define INSERTION_SORT 32
////////////////////////////////////////////////////////////////////////////////
// Selection sort used when depth gets too big or the number of elements drops
// below a threshold.
////////////////////////////////////////////////////////////////////////////////
__device__ void selection_sort(unsigned int *data, int left, int right) {
for (int i = left; i <= right; ++i) {
unsigned min_val = data[i];
int min_idx = i;
__device__ void selection_sort(unsigned int *data, int left, int right)
{
for (int i = left; i <= right; ++i) {
unsigned min_val = data[i];
int min_idx = i;
// Find the smallest value in the range [left, right].
for (int j = i + 1; j <= right; ++j) {
unsigned val_j = data[j];
// Find the smallest value in the range [left, right].
for (int j = i + 1; j <= right; ++j) {
unsigned val_j = data[j];
if (val_j < min_val) {
min_idx = j;
min_val = val_j;
}
if (val_j < min_val) {
min_idx = j;
min_val = val_j;
}
}
// Swap the values.
if (i != min_idx) {
data[min_idx] = data[i];
data[i] = min_val;
}
}
// Swap the values.
if (i != min_idx) {
data[min_idx] = data[i];
data[i] = min_val;
}
}
}
////////////////////////////////////////////////////////////////////////////////
// Very basic quicksort algorithm, recursively launching the next level.
////////////////////////////////////////////////////////////////////////////////
__global__ void cdp_simple_quicksort(unsigned int *data, int left, int right,
int depth) {
// If we're too deep or there are few elements left, we use an insertion
// sort...
if (depth >= MAX_DEPTH || right - left <= INSERTION_SORT) {
selection_sort(data, left, right);
return;
}
unsigned int *lptr = data + left;
unsigned int *rptr = data + right;
unsigned int pivot = data[(left + right) / 2];
// Do the partitioning.
while (lptr <= rptr) {
// Find the next left- and right-hand values to swap
unsigned int lval = *lptr;
unsigned int rval = *rptr;
// Move the left pointer as long as the pointed element is smaller than the
// pivot.
while (lval < pivot) {
lptr++;
lval = *lptr;
__global__ void cdp_simple_quicksort(unsigned int *data, int left, int right, int depth)
{
// If we're too deep or there are few elements left, we use an insertion
// sort...
if (depth >= MAX_DEPTH || right - left <= INSERTION_SORT) {
selection_sort(data, left, right);
return;
}
// Move the right pointer as long as the pointed element is larger than the
// pivot.
while (rval > pivot) {
rptr--;
rval = *rptr;
unsigned int *lptr = data + left;
unsigned int *rptr = data + right;
unsigned int pivot = data[(left + right) / 2];
// Do the partitioning.
while (lptr <= rptr) {
// Find the next left- and right-hand values to swap
unsigned int lval = *lptr;
unsigned int rval = *rptr;
// Move the left pointer as long as the pointed element is smaller than the
// pivot.
while (lval < pivot) {
lptr++;
lval = *lptr;
}
// Move the right pointer as long as the pointed element is larger than the
// pivot.
while (rval > pivot) {
rptr--;
rval = *rptr;
}
// If the swap points are valid, do the swap!
if (lptr <= rptr) {
*lptr++ = rval;
*rptr-- = lval;
}
}
// If the swap points are valid, do the swap!
if (lptr <= rptr) {
*lptr++ = rval;
*rptr-- = lval;
// Now the recursive part
int nright = rptr - data;
int nleft = lptr - data;
// Launch a new block to sort the left part.
if (left < (rptr - data)) {
cudaStream_t s;
cudaStreamCreateWithFlags(&s, cudaStreamNonBlocking);
cdp_simple_quicksort<<<1, 1, 0, s>>>(data, left, nright, depth + 1);
cudaStreamDestroy(s);
}
}
// Now the recursive part
int nright = rptr - data;
int nleft = lptr - data;
// Launch a new block to sort the left part.
if (left < (rptr - data)) {
cudaStream_t s;
cudaStreamCreateWithFlags(&s, cudaStreamNonBlocking);
cdp_simple_quicksort<<<1, 1, 0, s>>>(data, left, nright, depth + 1);
cudaStreamDestroy(s);
}
// Launch a new block to sort the right part.
if ((lptr - data) < right) {
cudaStream_t s1;
cudaStreamCreateWithFlags(&s1, cudaStreamNonBlocking);
cdp_simple_quicksort<<<1, 1, 0, s1>>>(data, nleft, right, depth + 1);
cudaStreamDestroy(s1);
}
// Launch a new block to sort the right part.
if ((lptr - data) < right) {
cudaStream_t s1;
cudaStreamCreateWithFlags(&s1, cudaStreamNonBlocking);
cdp_simple_quicksort<<<1, 1, 0, s1>>>(data, nleft, right, depth + 1);
cudaStreamDestroy(s1);
}
}
////////////////////////////////////////////////////////////////////////////////
// Call the quicksort kernel from the host.
////////////////////////////////////////////////////////////////////////////////
void run_qsort(unsigned int *data, unsigned int nitems) {
// Prepare CDP for the max depth 'MAX_DEPTH'.
void run_qsort(unsigned int *data, unsigned int nitems)
{
// Prepare CDP for the max depth 'MAX_DEPTH'.
// Launch on device
int left = 0;
int right = nitems - 1;
std::cout << "Launching kernel on the GPU" << std::endl;
cdp_simple_quicksort<<<1, 1>>>(data, left, right, 0);
checkCudaErrors(cudaDeviceSynchronize());
// Launch on device
int left = 0;
int right = nitems - 1;
std::cout << "Launching kernel on the GPU" << std::endl;
cdp_simple_quicksort<<<1, 1>>>(data, left, right, 0);
checkCudaErrors(cudaDeviceSynchronize());
}
////////////////////////////////////////////////////////////////////////////////
// Initialize data on the host.
////////////////////////////////////////////////////////////////////////////////
void initialize_data(unsigned int *dst, unsigned int nitems) {
// Fixed seed for illustration
srand(2047);
void initialize_data(unsigned int *dst, unsigned int nitems)
{
// Fixed seed for illustration
srand(2047);
// Fill dst with random values
for (unsigned i = 0; i < nitems; i++) dst[i] = rand() % nitems;
// Fill dst with random values
for (unsigned i = 0; i < nitems; i++)
dst[i] = rand() % nitems;
}
////////////////////////////////////////////////////////////////////////////////
// Verify the results.
////////////////////////////////////////////////////////////////////////////////
void check_results(int n, unsigned int *results_d) {
unsigned int *results_h = new unsigned[n];
checkCudaErrors(cudaMemcpy(results_h, results_d, n * sizeof(unsigned),
cudaMemcpyDeviceToHost));
void check_results(int n, unsigned int *results_d)
{
unsigned int *results_h = new unsigned[n];
checkCudaErrors(cudaMemcpy(results_h, results_d, n * sizeof(unsigned), cudaMemcpyDeviceToHost));
for (int i = 1; i < n; ++i)
if (results_h[i - 1] > results_h[i]) {
std::cout << "Invalid item[" << i - 1 << "]: " << results_h[i - 1]
<< " greater than " << results_h[i] << std::endl;
exit(EXIT_FAILURE);
}
for (int i = 1; i < n; ++i)
if (results_h[i - 1] > results_h[i]) {
std::cout << "Invalid item[" << i - 1 << "]: " << results_h[i - 1] << " greater than " << results_h[i]
<< std::endl;
exit(EXIT_FAILURE);
}
std::cout << "OK" << std::endl;
delete[] results_h;
std::cout << "OK" << std::endl;
delete[] results_h;
}
////////////////////////////////////////////////////////////////////////////////
// Main entry point.
////////////////////////////////////////////////////////////////////////////////
int main(int argc, char **argv) {
int num_items = 128;
bool verbose = false;
int main(int argc, char **argv)
{
int num_items = 128;
bool verbose = false;
if (checkCmdLineFlag(argc, (const char **)argv, "help") ||
checkCmdLineFlag(argc, (const char **)argv, "h")) {
std::cerr << "Usage: " << argv[0]
<< " num_items=<num_items>\twhere num_items is the number of "
"items to sort"
<< std::endl;
exit(EXIT_SUCCESS);
}
if (checkCmdLineFlag(argc, (const char **)argv, "v")) {
verbose = true;
}
if (checkCmdLineFlag(argc, (const char **)argv, "num_items")) {
num_items = getCmdLineArgumentInt(argc, (const char **)argv, "num_items");
if (num_items < 1) {
std::cerr << "ERROR: num_items has to be greater than 1" << std::endl;
exit(EXIT_FAILURE);
if (checkCmdLineFlag(argc, (const char **)argv, "help") || checkCmdLineFlag(argc, (const char **)argv, "h")) {
std::cerr << "Usage: " << argv[0]
<< " num_items=<num_items>\twhere num_items is the number of "
"items to sort"
<< std::endl;
exit(EXIT_SUCCESS);
}
}
// Find/set device and get device properties
int device = -1;
cudaDeviceProp deviceProp;
device = findCudaDevice(argc, (const char **)argv);
checkCudaErrors(cudaGetDeviceProperties(&deviceProp, device));
if (checkCmdLineFlag(argc, (const char **)argv, "v")) {
verbose = true;
}
if (!(deviceProp.major > 3 ||
(deviceProp.major == 3 && deviceProp.minor >= 5))) {
printf("GPU %d - %s does not support CUDA Dynamic Parallelism\n Exiting.",
device, deviceProp.name);
exit(EXIT_WAIVED);
}
if (checkCmdLineFlag(argc, (const char **)argv, "num_items")) {
num_items = getCmdLineArgumentInt(argc, (const char **)argv, "num_items");
// Create input data
unsigned int *h_data = 0;
unsigned int *d_data = 0;
if (num_items < 1) {
std::cerr << "ERROR: num_items has to be greater than 1" << std::endl;
exit(EXIT_FAILURE);
}
}
// Allocate CPU memory and initialize data.
std::cout << "Initializing data:" << std::endl;
h_data = (unsigned int *)malloc(num_items * sizeof(unsigned int));
initialize_data(h_data, num_items);
// Find/set device and get device properties
int device = -1;
cudaDeviceProp deviceProp;
device = findCudaDevice(argc, (const char **)argv);
checkCudaErrors(cudaGetDeviceProperties(&deviceProp, device));
if (verbose) {
for (int i = 0; i < num_items; i++)
std::cout << "Data [" << i << "]: " << h_data[i] << std::endl;
}
if (!(deviceProp.major > 3 || (deviceProp.major == 3 && deviceProp.minor >= 5))) {
printf("GPU %d - %s does not support CUDA Dynamic Parallelism\n Exiting.", device, deviceProp.name);
exit(EXIT_WAIVED);
}
// Allocate GPU memory.
checkCudaErrors(
cudaMalloc((void **)&d_data, num_items * sizeof(unsigned int)));
checkCudaErrors(cudaMemcpy(d_data, h_data, num_items * sizeof(unsigned int),
cudaMemcpyHostToDevice));
// Create input data
unsigned int *h_data = 0;
unsigned int *d_data = 0;
// Execute
std::cout << "Running quicksort on " << num_items << " elements" << std::endl;
run_qsort(d_data, num_items);
// Allocate CPU memory and initialize data.
std::cout << "Initializing data:" << std::endl;
h_data = (unsigned int *)malloc(num_items * sizeof(unsigned int));
initialize_data(h_data, num_items);
// Check result
std::cout << "Validating results: ";
check_results(num_items, d_data);
if (verbose) {
for (int i = 0; i < num_items; i++)
std::cout << "Data [" << i << "]: " << h_data[i] << std::endl;
}
free(h_data);
checkCudaErrors(cudaFree(d_data));
// Allocate GPU memory.
checkCudaErrors(cudaMalloc((void **)&d_data, num_items * sizeof(unsigned int)));
checkCudaErrors(cudaMemcpy(d_data, h_data, num_items * sizeof(unsigned int), cudaMemcpyHostToDevice));
exit(EXIT_SUCCESS);
// Execute
std::cout << "Running quicksort on " << num_items << " elements" << std::endl;
run_qsort(d_data, num_items);
// Check result
std::cout << "Validating results: ";
check_results(num_items, d_data);
free(h_data);
checkCudaErrors(cudaFree(d_data));
exit(EXIT_SUCCESS);
}

27
Samples/3_CUDA_Features/cudaCompressibleMemory/compMalloc.cpp
View File

@@ -25,11 +25,11 @@
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include <stdio.h>
#include <string.h>
#include <helper_cuda.h>
#include <cuda.h>
#include <cuda_runtime_api.h>
#include <helper_cuda.h>
#include <stdio.h>
#include <string.h>
cudaError_t setProp(CUmemAllocationProp *prop, bool UseCompressibleMemory)
{
@@ -38,9 +38,9 @@ cudaError_t setProp(CUmemAllocationProp *prop, bool UseCompressibleMemory)
return cudaErrorMemoryAllocation;
memset(prop, 0, sizeof(CUmemAllocationProp));
prop->type = CU_MEM_ALLOCATION_TYPE_PINNED;
prop->type = CU_MEM_ALLOCATION_TYPE_PINNED;
prop->location.type = CU_MEM_LOCATION_TYPE_DEVICE;
prop->location.id = currentDevice;
prop->location.id = currentDevice;
if (UseCompressibleMemory)
prop->allocFlags.compressionType = CU_MEM_ALLOCATION_COMP_GENERIC;
@@ -51,13 +51,12 @@ cudaError_t setProp(CUmemAllocationProp *prop, bool UseCompressibleMemory)
cudaError_t allocateCompressible(void **adr, size_t size, bool UseCompressibleMemory)
{
CUmemAllocationProp prop = {};
cudaError_t err = setProp(&prop, UseCompressibleMemory);
cudaError_t err = setProp(&prop, UseCompressibleMemory);
if (err != cudaSuccess)
return err;
size_t granularity = 0;
if (cuMemGetAllocationGranularity(&granularity, &prop,
CU_MEM_ALLOC_GRANULARITY_MINIMUM) != CUDA_SUCCESS)
if (cuMemGetAllocationGranularity(&granularity, &prop, CU_MEM_ALLOC_GRANULARITY_MINIMUM) != CUDA_SUCCESS)
return cudaErrorMemoryAllocation;
size = ((size - 1) / granularity + 1) * granularity;
CUdeviceptr dptr;
@@ -85,9 +84,9 @@ cudaError_t allocateCompressible(void **adr, size_t size, bool UseCompressibleMe
return cudaErrorMemoryAllocation;
CUmemAccessDesc accessDescriptor;
accessDescriptor.location.id = prop.location.id;
accessDescriptor.location.id = prop.location.id;
accessDescriptor.location.type = prop.location.type;
accessDescriptor.flags = CU_MEM_ACCESS_FLAGS_PROT_READWRITE;
accessDescriptor.flags = CU_MEM_ACCESS_FLAGS_PROT_READWRITE;
if (cuMemSetAccess(dptr, size, &accessDescriptor, 1) != CUDA_SUCCESS)
return cudaErrorMemoryAllocation;
@@ -99,20 +98,18 @@ cudaError_t allocateCompressible(void **adr, size_t size, bool UseCompressibleMe
cudaError_t freeCompressible(void *ptr, size_t size, bool UseCompressibleMemory)
{
CUmemAllocationProp prop = {};
cudaError_t err = setProp(&prop, UseCompressibleMemory);
cudaError_t err = setProp(&prop, UseCompressibleMemory);
if (err != cudaSuccess)
return err;
size_t granularity = 0;
if (cuMemGetAllocationGranularity(&granularity, &prop,
CU_MEM_ALLOC_GRANULARITY_MINIMUM) != CUDA_SUCCESS)
if (cuMemGetAllocationGranularity(&granularity, &prop, CU_MEM_ALLOC_GRANULARITY_MINIMUM) != CUDA_SUCCESS)
return cudaErrorMemoryAllocation;
size = ((size - 1) / granularity + 1) * granularity;
if (ptr == NULL)
return cudaSuccess;
if (cuMemUnmap((CUdeviceptr)ptr, size) != CUDA_SUCCESS ||
cuMemAddressFree((CUdeviceptr)ptr, size) != CUDA_SUCCESS)
if (cuMemUnmap((CUdeviceptr)ptr, size) != CUDA_SUCCESS || cuMemAddressFree((CUdeviceptr)ptr, size) != CUDA_SUCCESS)
return cudaErrorInvalidValue;
return cudaSuccess;
}

100
Samples/3_CUDA_Features/cudaCompressibleMemory/saxpy.cu
View File

@@ -27,53 +27,54 @@
//
// This sample uses the compressible memory allocation if device supports it
// and performs saxpy on it.
// Compressible memory may give better performance if the data is amenable to
// and performs saxpy on it.
// Compressible memory may give better performance if the data is amenable to
// compression.
#include <stdio.h>
#include <cuda.h>
#include <stdio.h>
#define CUDA_DRIVER_API
#include "helper_cuda.h"
#include "compMalloc.h"
#include "helper_cuda.h"
__global__ void saxpy(const float a, const float4 *x, const float4 *y, float4 *z, const size_t n)
{
for (size_t i = blockIdx.x * blockDim.x + threadIdx.x; i < n; i += gridDim.x * blockDim.x)
{
for (size_t i = blockIdx.x * blockDim.x + threadIdx.x; i < n; i += gridDim.x * blockDim.x) {
const float4 x4 = x[i];
const float4 y4 = y[i];
z[i] = make_float4(a * x4.x + y4.x, a * x4.y + y4.y,
a * x4.z + y4.z, a * x4.w + y4.w);
z[i] = make_float4(a * x4.x + y4.x, a * x4.y + y4.y, a * x4.z + y4.z, a * x4.w + y4.w);
}
}
__global__ void init(float4 *x, float4 *y, const float val, const size_t n)
{
const float4 val4 = make_float4(val, val, val, val);
for (size_t i = blockIdx.x * blockDim.x + threadIdx.x; i < n; i += gridDim.x * blockDim.x)
{
for (size_t i = blockIdx.x * blockDim.x + threadIdx.x; i < n; i += gridDim.x * blockDim.x) {
x[i] = y[i] = val4;
}
}
void launchSaxpy(const float a, float4 *x, float4 *y, float4 *z, const size_t n, const float init_val, const bool compressibleZbuf)
void launchSaxpy(const float a,
float4 *x,
float4 *y,
float4 *z,
const size_t n,
const float init_val,
const bool compressibleZbuf)
{
cudaEvent_t start, stop;
float ms;
int blockSize;
int minGridSize;
dim3 threads, blocks;
float ms;
int blockSize;
int minGridSize;
dim3 threads, blocks;
if (!compressibleZbuf)
{
if (!compressibleZbuf) {
// We are on config where compressible buffer can only be initialized through cudaMemcpy
// hence, x & y buffers are allocated as compressible and initialized via cudaMemcpy
// whereas z buffer is allocated as non-compressible.
float4 *h_x = (float4 *) malloc(sizeof(float4) * n);
float4 *h_y = (float4 *) malloc(sizeof(float4) * n);
for (int i = 0; i < n; i++)
{
float4 *h_x = (float4 *)malloc(sizeof(float4) * n);
float4 *h_y = (float4 *)malloc(sizeof(float4) * n);
for (int i = 0; i < n; i++) {
h_x[i].x = h_x[i].y = h_x[i].z = h_x[i].w = init_val;
h_y[i].x = h_y[i].y = h_y[i].z = h_y[i].w = init_val;
}
@@ -82,15 +83,14 @@ void launchSaxpy(const float a, float4 *x, float4 *y, float4 *z, const size_t n,
free(h_x);
free(h_y);
}
else
{
checkCudaErrors(cudaOccupancyMaxPotentialBlockSize(&minGridSize, &blockSize, (void*)init));
else {
checkCudaErrors(cudaOccupancyMaxPotentialBlockSize(&minGridSize, &blockSize, (void *)init));
threads = dim3(blockSize, 1, 1);
blocks = dim3(minGridSize, 1, 1);
init<<<blocks, threads>>>(x, y, init_val, n);
}
checkCudaErrors(cudaOccupancyMaxPotentialBlockSize(&minGridSize, &blockSize, (void*)saxpy));
checkCudaErrors(cudaOccupancyMaxPotentialBlockSize(&minGridSize, &blockSize, (void *)saxpy));
threads = dim3(blockSize, 1, 1);
blocks = dim3(minGridSize, 1, 1);
@@ -103,39 +103,39 @@ void launchSaxpy(const float a, float4 *x, float4 *y, float4 *z, const size_t n,
checkCudaErrors(cudaEventElapsedTime(&ms, start, stop));
const size_t size = n * sizeof(float4);
printf("Running saxpy with %d blocks x %d threads = %.3f ms %.3f TB/s\n", blocks.x, threads.x, ms, (size*3)/ms/1e9);
printf("Running saxpy with %d blocks x %d threads = %.3f ms %.3f TB/s\n",
blocks.x,
threads.x,
ms,
(size * 3) / ms / 1e9);
}
int main(int argc, char **argv)
{
const size_t n = 10485760;
if (checkCmdLineFlag(argc, (const char **)argv, "help") ||
checkCmdLineFlag(argc, (const char **)argv, "?")) {
if (checkCmdLineFlag(argc, (const char **)argv, "help") || checkCmdLineFlag(argc, (const char **)argv, "?")) {
printf("Usage -device=n (n >= 0 for deviceID)\n");
exit(EXIT_SUCCESS);
}
findCudaDevice(argc, (const char**)argv);
findCudaDevice(argc, (const char **)argv);
CUdevice currentDevice;
checkCudaErrors(cuCtxGetDevice(&currentDevice));
// Check that the selected device supports virtual memory management
int vmm_supported = -1;
checkCudaErrors(cuDeviceGetAttribute(&vmm_supported,
CU_DEVICE_ATTRIBUTE_VIRTUAL_ADDRESS_MANAGEMENT_SUPPORTED,
currentDevice));
checkCudaErrors(
cuDeviceGetAttribute(&vmm_supported, CU_DEVICE_ATTRIBUTE_VIRTUAL_ADDRESS_MANAGEMENT_SUPPORTED, currentDevice));
if (vmm_supported == 0) {
printf("Device %d doesn't support Virtual Memory Management, waiving the execution.\n", currentDevice);
exit(EXIT_WAIVED);
}
int isCompressionAvailable;
checkCudaErrors(cuDeviceGetAttribute(&isCompressionAvailable,
CU_DEVICE_ATTRIBUTE_GENERIC_COMPRESSION_SUPPORTED,
currentDevice));
if (isCompressionAvailable == 0)
{
checkCudaErrors(cuDeviceGetAttribute(
&isCompressionAvailable, CU_DEVICE_ATTRIBUTE_GENERIC_COMPRESSION_SUPPORTED, currentDevice));
if (isCompressionAvailable == 0) {
printf("Device %d doesn't support Generic memory compression, waiving the execution.\n", currentDevice);
exit(EXIT_WAIVED);
}
@@ -143,39 +143,33 @@ int main(int argc, char **argv)
printf("Generic memory compression support is available\n");
int major, minor;
checkCudaErrors(cuDeviceGetAttribute(&major,
CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR,
currentDevice));
checkCudaErrors(cuDeviceGetAttribute(&minor,
CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR,
currentDevice));
float4 *x, *y, *z;
checkCudaErrors(cuDeviceGetAttribute(&major, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR, currentDevice));
checkCudaErrors(cuDeviceGetAttribute(&minor, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR, currentDevice));
float4 *x, *y, *z;
const size_t size = n * sizeof(float4);
// Allocating compressible memory
checkCudaErrors(allocateCompressible((void **)&x, size, true));
checkCudaErrors(allocateCompressible((void **)&y, size, true));
bool compressibleZbuf = 0;
if ((major == 8 && minor == 0) || (major == 8 && minor == 6))
{
// On SM 8.0 and 8.6 GPUs compressible buffer can only be initialized
if ((major == 8 && minor == 0) || (major == 8 && minor == 6)) {
// On SM 8.0 and 8.6 GPUs compressible buffer can only be initialized
// through cudaMemcpy.
printf("allocating non-compressible Z buffer\n");
checkCudaErrors(allocateCompressible((void **)&z, size, false));
compressibleZbuf = 0;
}
else
{
else {
checkCudaErrors(allocateCompressible((void **)&z, size, true));
compressibleZbuf = 1;
}
printf("Running saxpy on %zu bytes of Compressible memory\n", size);
const float a = 1.0f;
const float a = 1.0f;
const float init_val = 1.0f;
launchSaxpy(a, x, y, z, n, init_val, compressibleZbuf);
checkCudaErrors(freeCompressible(x, size, true));
checkCudaErrors(freeCompressible(y, size, true));
checkCudaErrors(freeCompressible(z, size, true));
@@ -193,6 +187,6 @@ int main(int argc, char **argv)
checkCudaErrors(freeCompressible(z, size, false));
printf("\nNOTE: The CUDA Samples are not meant for performance measurements. "
"Results may vary when GPU Boost is enabled.\n");
"Results may vary when GPU Boost is enabled.\n");
return EXIT_SUCCESS;
}
}

821
Samples/3_CUDA_Features/cudaTensorCoreGemm/cudaTensorCoreGemm.cu
View File

@@ -112,7 +112,7 @@
// Implementation constants.
#define WARPS_PER_BLOCK 8
#define WARPS_PER_BLOCK 8
#define THREADS_PER_BLOCK (WARP_SIZE * WARPS_PER_BLOCK)
#if SHARED_MEMORY_LIMIT_64K
@@ -129,10 +129,10 @@
#define CHUNK_K 8
#endif
#define CHUNK_LINE_BYTES (CHUNK_K * K * sizeof(half))
#define WARP_COPY_BYTES (WARP_SIZE * sizeof(int4))
#define CHUNK_LINE_BYTES (CHUNK_K * K * sizeof(half))
#define WARP_COPY_BYTES (WARP_SIZE * sizeof(int4))
#define CHUNK_COPY_LINES_PER_WARP (WARP_COPY_BYTES / CHUNK_LINE_BYTES)
#define CHUNK_COPY_LINE_LANES (WARP_SIZE / CHUNK_COPY_LINES_PER_WARP)
#define CHUNK_COPY_LINE_LANES (WARP_SIZE / CHUNK_COPY_LINES_PER_WARP)
#define BLOCK_ROW_WARPS 2
#define BLOCK_COL_WARPS 4
@@ -162,237 +162,220 @@
// we must keep each row and column 256-bit aligned, as required by nvcuda::wmma::load_matrix_sync.
#define SKEW_HALF 16
#define checkKernelErrors(expr) \
do { \
expr; \
\
cudaError_t __err = cudaGetLastError(); \
if (__err != cudaSuccess) { \
printf("Line %d: '%s' failed: %s\n", __LINE__, #expr, \
cudaGetErrorString(__err)); \
abort(); \
} \
} while (0)
#define checkKernelErrors(expr) \
do { \
expr; \
\
cudaError_t __err = cudaGetLastError(); \
if (__err != cudaSuccess) { \
printf("Line %d: '%s' failed: %s\n", __LINE__, #expr, cudaGetErrorString(__err)); \
abort(); \
} \
} while (0)
using namespace nvcuda;
__host__ void init_host_matrices(half *a, half *b, float *c) {
for (int i = 0; i < M_GLOBAL; i++) {
for (int j = 0; j < K_GLOBAL; j++) {
a[i * K_GLOBAL + j] = (half)(rand() % 3);
__host__ void init_host_matrices(half *a, half *b, float *c)
{
for (int i = 0; i < M_GLOBAL; i++) {
for (int j = 0; j < K_GLOBAL; j++) {
a[i * K_GLOBAL + j] = (half)(rand() % 3);
}
}
}
for (int i = 0; i < N_GLOBAL; i++) {
for (int j = 0; j < K_GLOBAL; j++) {
b[i * K_GLOBAL + j] = (half)(rand() % 3);
for (int i = 0; i < N_GLOBAL; i++) {
for (int j = 0; j < K_GLOBAL; j++) {
b[i * K_GLOBAL + j] = (half)(rand() % 3);
}
}
}
for (int t = 0; t < M_GLOBAL * N_GLOBAL; t++) {
c[t] = static_cast<float>(rand() % 3);
}
for (int t = 0; t < M_GLOBAL * N_GLOBAL; t++) {
c[t] = static_cast<float>(rand() % 3);
}
}
__global__ void compute_gemm(const half *A, const half *B, const float *C,
float *D, float alpha, float beta) {
extern __shared__ half shmem[][CHUNK_K * K + SKEW_HALF];
__global__ void compute_gemm(const half *A, const half *B, const float *C, float *D, float alpha, float beta)
{
extern __shared__ half shmem[][CHUNK_K * K + SKEW_HALF];
// Warp and lane identification.
const unsigned int warpId = threadIdx.x / WARP_SIZE;
const unsigned int laneId = threadIdx.x % WARP_SIZE;
// Warp and lane identification.
const unsigned int warpId = threadIdx.x / WARP_SIZE;
const unsigned int laneId = threadIdx.x % WARP_SIZE;
// Offset in shared memory from which the B matrix is stored.
const size_t shmem_idx_b_off = BLOCK_COL_TILES * M;
// Offset in shared memory from which the B matrix is stored.
const size_t shmem_idx_b_off = BLOCK_COL_TILES * M;
// This pointer is used to access the C and D matrix tiles this warp computes.
float *shmem_warp_tile_ptr = (float *)&shmem[0][0] +
(warpId / 2) * SHMEM_STRIDE * K * 2 +
(warpId % 2) * SHMEM_OFFSET;
// This pointer is used to access the C and D matrix tiles this warp computes.
float *shmem_warp_tile_ptr =
(float *)&shmem[0][0] + (warpId / 2) * SHMEM_STRIDE * K * 2 + (warpId % 2) * SHMEM_OFFSET;
// This pointer is used to stream the C and D matrices block-wide tile to and
// from shared memory.
float *shmem_warp_stream_ptr =
(float *)&shmem[0][0] + warpId * SHMEM_STRIDE * K;
// This pointer is used to stream the C and D matrices block-wide tile to and
// from shared memory.
float *shmem_warp_stream_ptr = (float *)&shmem[0][0] + warpId * SHMEM_STRIDE * K;
// Adjust the beta scaler, as it'll be multiplied by alpha at the end of
// each tile computation. Technically this is not generally correct (may
// result in a loss of precision). Zero still needs to be specially handled
// though.
beta /= alpha;
// Adjust the beta scaler, as it'll be multiplied by alpha at the end of
// each tile computation. Technically this is not generally correct (may
// result in a loss of precision). Zero still needs to be specially handled
// though.
beta /= alpha;
// Each CTA slides along the 128 x 128 tiles from the top left corner of the
// matrix to the right and down, and selects the next tile to compute. Once
// there's no such tile, all warps in this CTA exit.
for (unsigned int block_pos = blockIdx.x;; block_pos += gridDim.x) {
const unsigned int block_tile_i =
((block_pos * BLOCK_ROW_TILES) / N_TILES) * (BLOCK_COL_TILES);
const unsigned int block_tile_j = (block_pos * BLOCK_COL_TILES) % N_TILES;
// Each CTA slides along the 128 x 128 tiles from the top left corner of the
// matrix to the right and down, and selects the next tile to compute. Once
// there's no such tile, all warps in this CTA exit.
for (unsigned int block_pos = blockIdx.x;; block_pos += gridDim.x) {
const unsigned int block_tile_i = ((block_pos * BLOCK_ROW_TILES) / N_TILES) * (BLOCK_COL_TILES);
const unsigned int block_tile_j = (block_pos * BLOCK_COL_TILES) % N_TILES;
// Stop when there are no more D matrix tiles to compute in this CTA.
if (block_tile_i >= M_TILES) {
break;
}
// This warp's pointer to the C matrix data to copy memory from to shared
// memory.
const size_t gmem_idx =
(block_tile_i + warpId) * M * GLOBAL_MEM_STRIDE + block_tile_j * N;
const float *src_gmem_warp_stream_ptr = &C[gmem_idx];
// Stream multiple C tiles to shared memory.
#pragma unroll
for (int i = 0; i < K; i++) {
typedef int4 copy_t;
*((copy_t *)(shmem_warp_stream_ptr + SHMEM_STRIDE * i) + laneId) =
*((copy_t *)(src_gmem_warp_stream_ptr + GLOBAL_MEM_STRIDE * i) +
laneId);
}
__syncthreads();
// These fragments will accumulate the result of A and B matrix fragment
// multiplications along the K_GLOBAL dimension.
wmma::fragment<wmma::accumulator, M, N, K, float> c[WARP_COL_TILES]
[WARP_ROW_TILES];
// Load the C matrix tiles into fragments from shared memory.
#pragma unroll
for (int i = 0; i < WARP_COL_TILES; i++) {
#pragma unroll
for (int j = 0; j < WARP_ROW_TILES; j++) {
const float *tile_ptr =
shmem_warp_tile_ptr + i * SHMEM_STRIDE * K + j * N;
wmma::load_matrix_sync(c[i][j], tile_ptr, SHMEM_STRIDE, C_LAYOUT);
}
}
__syncthreads();
// Scale the C matrix.
#pragma unroll
for (int i = 0; i < WARP_COL_TILES; i++) {
#pragma unroll
for (int j = 0; j < WARP_ROW_TILES; j++) {
#pragma unroll
for (int t = 0; t < c[i][j].num_elements; t++) {
c[i][j].x[t] *= beta;
// Stop when there are no more D matrix tiles to compute in this CTA.
if (block_tile_i >= M_TILES) {
break;
}
}
}
// Select what warp copies what matrix to shared memory.
// Warps 0-3 copy the A matrix, warps 4-7 copy the B matrix.
const half *warp_ptr = (warpId < 4) ? (&A[block_tile_i * M * K_GLOBAL] +
M * K_GLOBAL * (warpId % 4) * 2)
: (&B[block_tile_j * N * K_GLOBAL] +
N * K_GLOBAL * (warpId % 4) * 2);
// This warp's pointer to the C matrix data to copy memory from to shared
// memory.
const size_t gmem_idx = (block_tile_i + warpId) * M * GLOBAL_MEM_STRIDE + block_tile_j * N;
const float *src_gmem_warp_stream_ptr = &C[gmem_idx];
// Go through the global K dimension by a fixed step at a time.
// Stream multiple C tiles to shared memory.
#pragma unroll
for (int tile_k = 0; tile_k < K_TILES; tile_k += CHUNK_K) {
// Copy slices of the A and B matrices to shared memory.
// The first half of the warps in the CTA copy the A matrix, the rest copy
// the B matrix.
size_t shmem_idx =
warpId < (WARPS_PER_BLOCK / 2)
? (M * (warpId % (WARPS_PER_BLOCK / 2)) * 2)
: (N * (warpId % (WARPS_PER_BLOCK / 2)) * 2 + shmem_idx_b_off);
for (int i = 0; i < K; i++) {
typedef int4 copy_t;
// First half of the warp copies the first row / column of the matrix,
// the second half of the warp copies the next.
int4 *lane_ptr = (int4 *)(warp_ptr + tile_k * K +
(laneId / CHUNK_COPY_LINE_LANES) * K_GLOBAL) +
(laneId % CHUNK_COPY_LINE_LANES);
*((copy_t *)(shmem_warp_stream_ptr + SHMEM_STRIDE * i) + laneId) =
*((copy_t *)(src_gmem_warp_stream_ptr + GLOBAL_MEM_STRIDE * i) + laneId);
}
// Shift the second half of the warp to the next row / column in the
// shared memory.
shmem_idx += laneId / CHUNK_COPY_LINE_LANES;
__syncthreads();
#pragma unroll
for (int i = 0; i < ((WARP_SIZE / 2) / CHUNK_COPY_LINES_PER_WARP) * 2;
i++) {
// Copy 16 bytes at once in each lane.
*((int4 *)&shmem[shmem_idx][0] + (laneId % CHUNK_COPY_LINE_LANES)) =
*lane_ptr;
// Advance the global memory pointer and the shared memory index.
lane_ptr =
(int4 *)((half *)lane_ptr + K_GLOBAL * CHUNK_COPY_LINES_PER_WARP);
shmem_idx += CHUNK_COPY_LINES_PER_WARP;
}
__syncthreads();
// Compute a grid of C matrix tiles in each warp.
#pragma unroll
for (int k_step = 0; k_step < CHUNK_K; k_step++) {
wmma::fragment<wmma::matrix_a, M, N, K, half, wmma::row_major>
a[WARP_COL_TILES];
wmma::fragment<wmma::matrix_b, M, N, K, half, wmma::col_major>
b[WARP_ROW_TILES];
// These fragments will accumulate the result of A and B matrix fragment
// multiplications along the K_GLOBAL dimension.
wmma::fragment<wmma::accumulator, M, N, K, float> c[WARP_COL_TILES][WARP_ROW_TILES];
// Load the C matrix tiles into fragments from shared memory.
#pragma unroll
for (int i = 0; i < WARP_COL_TILES; i++) {
size_t shmem_idx_a = (warpId / 2) * M * 2 + (i * M);
const half *tile_ptr = &shmem[shmem_idx_a][k_step * K];
#pragma unroll
for (int j = 0; j < WARP_ROW_TILES; j++) {
const float *tile_ptr = shmem_warp_tile_ptr + i * SHMEM_STRIDE * K + j * N;
wmma::load_matrix_sync(a[i], tile_ptr, K * CHUNK_K + SKEW_HALF);
wmma::load_matrix_sync(c[i][j], tile_ptr, SHMEM_STRIDE, C_LAYOUT);
}
}
__syncthreads();
// Scale the C matrix.
#pragma unroll
for (int i = 0; i < WARP_COL_TILES; i++) {
#pragma unroll
for (int j = 0; j < WARP_ROW_TILES; j++) {
#pragma unroll
for (int t = 0; t < c[i][j].num_elements; t++) {
c[i][j].x[t] *= beta;
}
}
}
// Select what warp copies what matrix to shared memory.
// Warps 0-3 copy the A matrix, warps 4-7 copy the B matrix.
const half *warp_ptr = (warpId < 4) ? (&A[block_tile_i * M * K_GLOBAL] + M * K_GLOBAL * (warpId % 4) * 2)
: (&B[block_tile_j * N * K_GLOBAL] + N * K_GLOBAL * (warpId % 4) * 2);
// Go through the global K dimension by a fixed step at a time.
#pragma unroll
for (int tile_k = 0; tile_k < K_TILES; tile_k += CHUNK_K) {
// Copy slices of the A and B matrices to shared memory.
// The first half of the warps in the CTA copy the A matrix, the rest copy
// the B matrix.
size_t shmem_idx = warpId < (WARPS_PER_BLOCK / 2)
? (M * (warpId % (WARPS_PER_BLOCK / 2)) * 2)
: (N * (warpId % (WARPS_PER_BLOCK / 2)) * 2 + shmem_idx_b_off);
// First half of the warp copies the first row / column of the matrix,
// the second half of the warp copies the next.
int4 *lane_ptr = (int4 *)(warp_ptr + tile_k * K + (laneId / CHUNK_COPY_LINE_LANES) * K_GLOBAL)
+ (laneId % CHUNK_COPY_LINE_LANES);
// Shift the second half of the warp to the next row / column in the
// shared memory.
shmem_idx += laneId / CHUNK_COPY_LINE_LANES;
#pragma unroll
for (int j = 0; j < WARP_ROW_TILES; j++) {
if (i == 0) {
// Load the B matrix fragment once, because it is going to be
// reused against the other A matrix fragments.
size_t shmem_idx_b = shmem_idx_b_off +
(WARP_ROW_TILES * N) * (warpId % 2) +
(j * N);
const half *tile_ptr = &shmem[shmem_idx_b][k_step * K];
for (int i = 0; i < ((WARP_SIZE / 2) / CHUNK_COPY_LINES_PER_WARP) * 2; i++) {
// Copy 16 bytes at once in each lane.
*((int4 *)&shmem[shmem_idx][0] + (laneId % CHUNK_COPY_LINE_LANES)) = *lane_ptr;
wmma::load_matrix_sync(b[j], tile_ptr, K * CHUNK_K + SKEW_HALF);
// Advance the global memory pointer and the shared memory index.
lane_ptr = (int4 *)((half *)lane_ptr + K_GLOBAL * CHUNK_COPY_LINES_PER_WARP);
shmem_idx += CHUNK_COPY_LINES_PER_WARP;
}
wmma::mma_sync(c[i][j], a[i], b[j], c[i][j]);
}
__syncthreads();
// Compute a grid of C matrix tiles in each warp.
#pragma unroll
for (int k_step = 0; k_step < CHUNK_K; k_step++) {
wmma::fragment<wmma::matrix_a, M, N, K, half, wmma::row_major> a[WARP_COL_TILES];
wmma::fragment<wmma::matrix_b, M, N, K, half, wmma::col_major> b[WARP_ROW_TILES];
#pragma unroll
for (int i = 0; i < WARP_COL_TILES; i++) {
size_t shmem_idx_a = (warpId / 2) * M * 2 + (i * M);
const half *tile_ptr = &shmem[shmem_idx_a][k_step * K];
wmma::load_matrix_sync(a[i], tile_ptr, K * CHUNK_K + SKEW_HALF);
#pragma unroll
for (int j = 0; j < WARP_ROW_TILES; j++) {
if (i == 0) {
// Load the B matrix fragment once, because it is going to be
// reused against the other A matrix fragments.
size_t shmem_idx_b = shmem_idx_b_off + (WARP_ROW_TILES * N) * (warpId % 2) + (j * N);
const half *tile_ptr = &shmem[shmem_idx_b][k_step * K];
wmma::load_matrix_sync(b[j], tile_ptr, K * CHUNK_K + SKEW_HALF);
}
wmma::mma_sync(c[i][j], a[i], b[j], c[i][j]);
}
}
}
__syncthreads();
}
}
__syncthreads();
// Store the D fragments to shared memory.
#pragma unroll
for (int i = 0; i < WARP_COL_TILES; i++) {
#pragma unroll
for (int j = 0; j < WARP_ROW_TILES; j++) {
#pragma unroll
// Uniform, point-wise transformations of ALL fragment elements by ALL
// threads in the warp are well-defined even though element indices
// within fragment storage are not defined.
for (int t = 0; t < c[i][j].num_elements; t++)
c[i][j].x[t] *= alpha;
float *tile_ptr = shmem_warp_tile_ptr + i * SHMEM_STRIDE * K + j * N;
wmma::store_matrix_sync(tile_ptr, c[i][j], SHMEM_STRIDE, C_LAYOUT);
}
}
__syncthreads();
// Now that shared memory contains all the D tiles, stream them to global
// memory.
float *dst_gmem_warp_stream_ptr = &D[gmem_idx];
#pragma unroll
for (int i = 0; i < K; i++) {
*((int4 *)(dst_gmem_warp_stream_ptr + GLOBAL_MEM_STRIDE * i) + laneId) =
*((int4 *)(shmem_warp_stream_ptr + SHMEM_STRIDE * i) + laneId);
}
__syncthreads();
}
// Store the D fragments to shared memory.
#pragma unroll
for (int i = 0; i < WARP_COL_TILES; i++) {
#pragma unroll
for (int j = 0; j < WARP_ROW_TILES; j++) {
#pragma unroll
// Uniform, point-wise transformations of ALL fragment elements by ALL
// threads in the warp are well-defined even though element indices
// within fragment storage are not defined.
for (int t = 0; t < c[i][j].num_elements; t++) c[i][j].x[t] *= alpha;
float *tile_ptr = shmem_warp_tile_ptr + i * SHMEM_STRIDE * K + j * N;
wmma::store_matrix_sync(tile_ptr, c[i][j], SHMEM_STRIDE, C_LAYOUT);
}
}
__syncthreads();
// Now that shared memory contains all the D tiles, stream them to global
// memory.
float *dst_gmem_warp_stream_ptr = &D[gmem_idx];
#pragma unroll
for (int i = 0; i < K; i++) {
*((int4 *)(dst_gmem_warp_stream_ptr + GLOBAL_MEM_STRIDE * i) + laneId) =
*((int4 *)(shmem_warp_stream_ptr + SHMEM_STRIDE * i) + laneId);
}
__syncthreads();
}
}
// Performs an MxNxK GEMM (C=alpha*A*B + beta*C) assuming:
@@ -403,246 +386,232 @@ __global__ void compute_gemm(const half *A, const half *B, const float *C,
// designed for
// demonstration purposes only to show the CUDA WMMA API use without
// relying on availability of the shared memory.
__global__ void simple_wmma_gemm(half *a, half *b, float *c, float *d, int m_ld,
int n_ld, int k_ld, float alpha, float beta) {
// Leading dimensions. Packed with no transpositions.
int lda = k_ld;
int ldb = k_ld;
int ldc = n_ld;
__global__ void
simple_wmma_gemm(half *a, half *b, float *c, float *d, int m_ld, int n_ld, int k_ld, float alpha, float beta)
{
// Leading dimensions. Packed with no transpositions.
int lda = k_ld;
int ldb = k_ld;
int ldc = n_ld;
// Tile using a 2D grid
int warpM = (blockIdx.x * blockDim.x + threadIdx.x) / warpSize;
int warpN = (blockIdx.y * blockDim.y + threadIdx.y);
// Tile using a 2D grid
int warpM = (blockIdx.x * blockDim.x + threadIdx.x) / warpSize;
int warpN = (blockIdx.y * blockDim.y + threadIdx.y);
// Declare the fragments
wmma::fragment<wmma::matrix_a, WMMA_M, WMMA_N, WMMA_K, half, wmma::row_major>
a_frag;
wmma::fragment<wmma::matrix_b, WMMA_M, WMMA_N, WMMA_K, half, wmma::col_major>
b_frag;
wmma::fragment<wmma::accumulator, WMMA_M, WMMA_N, WMMA_K, float> acc_frag;
wmma::fragment<wmma::accumulator, WMMA_M, WMMA_N, WMMA_K, float> c_frag;
// Declare the fragments
wmma::fragment<wmma::matrix_a, WMMA_M, WMMA_N, WMMA_K, half, wmma::row_major> a_frag;
wmma::fragment<wmma::matrix_b, WMMA_M, WMMA_N, WMMA_K, half, wmma::col_major> b_frag;
wmma::fragment<wmma::accumulator, WMMA_M, WMMA_N, WMMA_K, float> acc_frag;
wmma::fragment<wmma::accumulator, WMMA_M, WMMA_N, WMMA_K, float> c_frag;
wmma::fill_fragment(acc_frag, 0.0f);
wmma::fill_fragment(acc_frag, 0.0f);
// Loop over k
for (int i = 0; i < k_ld; i += WMMA_K) {
int aCol = i;
int aRow = warpM * WMMA_M;
int bCol = warpN * N;
int bRow = i;
// Loop over k
for (int i = 0; i < k_ld; i += WMMA_K) {
int aCol = i;
int aRow = warpM * WMMA_M;
int bCol = warpN * N;
int bRow = i;
// Bounds checking
if (aRow < m_ld && aCol < k_ld && bRow < k_ld && bCol < n_ld) {
// Load the inputs
wmma::load_matrix_sync(a_frag, a + aCol + aRow * lda, lda);
wmma::load_matrix_sync(b_frag, b + bRow + bCol * ldb, ldb);
// Bounds checking
if (aRow < m_ld && aCol < k_ld && bRow < k_ld && bCol < n_ld) {
// Load the inputs
wmma::load_matrix_sync(a_frag, a + aCol + aRow * lda, lda);
wmma::load_matrix_sync(b_frag, b + bRow + bCol * ldb, ldb);
// Perform the matrix multiplication
wmma::mma_sync(acc_frag, a_frag, b_frag, acc_frag);
}
}
// Load in the current value of c, scale it by beta, and add this our result
// scaled by alpha
int cCol = warpN * WMMA_N;
int cRow = warpM * WMMA_M;
if (cRow < m_ld && cCol < n_ld) {
wmma::load_matrix_sync(c_frag, c + cCol + cRow * ldc, ldc,
wmma::mem_row_major);
for (int i = 0; i < c_frag.num_elements; i++) {
c_frag.x[i] = alpha * acc_frag.x[i] + beta * c_frag.x[i];
// Perform the matrix multiplication
wmma::mma_sync(acc_frag, a_frag, b_frag, acc_frag);
}
}
// Store the output
wmma::store_matrix_sync(d + cCol + cRow * ldc, c_frag, ldc,
wmma::mem_row_major);
}
}
// Load in the current value of c, scale it by beta, and add this our result
// scaled by alpha
int cCol = warpN * WMMA_N;
int cRow = warpM * WMMA_M;
__host__ void matMultiplyOnHost(half *A, half *B, float *C, float alpha,
float beta, int numARows, int numAColumns,
int numBRows, int numBColumns, int numCRows,
int numCColumns) {
for (int i = 0; i < numCRows; i++) {
for (int j = 0; j < numCColumns; j++) {
float temp = 0.0;
if (cRow < m_ld && cCol < n_ld) {
wmma::load_matrix_sync(c_frag, c + cCol + cRow * ldc, ldc, wmma::mem_row_major);
for (int k = 0; k < numAColumns; k++) {
temp += (float)A[i * numAColumns + k] * (float)B[j * numBRows + k];
}
for (int i = 0; i < c_frag.num_elements; i++) {
c_frag.x[i] = alpha * acc_frag.x[i] + beta * c_frag.x[i];
}
C[i * numCColumns + j] = temp * alpha + beta * C[i * numCColumns + j];
// Store the output
wmma::store_matrix_sync(d + cCol + cRow * ldc, c_frag, ldc, wmma::mem_row_major);
}
}
}
int main(int argc, char **argv) {
printf("Initializing...\n");
__host__ void matMultiplyOnHost(half *A,
half *B,
float *C,
float alpha,
float beta,
int numARows,
int numAColumns,
int numBRows,
int numBColumns,
int numCRows,
int numCColumns)
{
for (int i = 0; i < numCRows; i++) {
for (int j = 0; j < numCColumns; j++) {
float temp = 0.0;
int dev = findCudaDevice(argc, (const char **)argv);
for (int k = 0; k < numAColumns; k++) {
temp += (float)A[i * numAColumns + k] * (float)B[j * numBRows + k];
}
cudaDeviceProp deviceProp;
checkCudaErrors(cudaGetDeviceProperties(&deviceProp, dev));
// Tensor cores require a GPU of Volta (SM7X) architecture or higher.
if (deviceProp.major < 7) {
printf(
"cudaTensorCoreGemm requires SM 7.0 or higher to use Tensor "
"Cores. Exiting...\n");
exit(EXIT_WAIVED);
}
printf("M: %d (%d x %d)\n", M_GLOBAL, M, M_TILES);
printf("N: %d (%d x %d)\n", N_GLOBAL, N, N_TILES);
printf("K: %d (%d x %d)\n", K_GLOBAL, K, K_TILES);
half *A_h = NULL;
half *B_h = NULL;
float *C_h = NULL;
#if CPU_DEBUG
float *result_hD = NULL;
float *result_host = NULL;
#endif
A_h = (half *)malloc(sizeof(half) * M_GLOBAL * K_GLOBAL);
B_h = (half *)malloc(sizeof(half) * K_GLOBAL * N_GLOBAL);
C_h = (float *)malloc(sizeof(float) * M_GLOBAL * N_GLOBAL);
#if CPU_DEBUG
result_hD = (float *)malloc(sizeof(float) * M_GLOBAL * N_GLOBAL);
result_host = (float *)malloc(sizeof(float) * M_GLOBAL * N_GLOBAL);
#endif
half *A = NULL;
half *B = NULL;
float *C = NULL;
float *D = NULL;
checkCudaErrors(cudaMalloc(reinterpret_cast<void **>(&A),
sizeof(half) * M_GLOBAL * K_GLOBAL));
checkCudaErrors(cudaMalloc(reinterpret_cast<void **>(&B),
sizeof(half) * N_GLOBAL * K_GLOBAL));
checkCudaErrors(cudaMalloc(reinterpret_cast<void **>(&C),
sizeof(float) * M_GLOBAL * N_GLOBAL));
checkCudaErrors(cudaMalloc(reinterpret_cast<void **>(&D),
sizeof(float) * M_GLOBAL * N_GLOBAL));
assert(((unsigned long long)A) % 128 == 0);
assert(((unsigned long long)B) % 128 == 0);
assert(((unsigned long long)C) % 128 == 0);
assert(((unsigned long long)D) % 128 == 0);
init_host_matrices(A_h, B_h, C_h);
printf("Preparing data for GPU...\n");
checkCudaErrors(cudaMemcpy(A, A_h, sizeof(half) * M_GLOBAL * K_GLOBAL,
cudaMemcpyHostToDevice));
checkCudaErrors(cudaMemcpy(B, B_h, sizeof(half) * N_GLOBAL * K_GLOBAL,
cudaMemcpyHostToDevice));
checkCudaErrors(cudaMemcpy(C, C_h, sizeof(float) * M_GLOBAL * N_GLOBAL,
cudaMemcpyHostToDevice));
checkCudaErrors(cudaMemset(D, 0, sizeof(float) * M_GLOBAL * N_GLOBAL));
enum {
// Compute the right amount of shared memory to request.
// We need shared memory to hold per-CTA C and D matrix tiles, and to cache
// per-CTA chunks
// of the A and B matrices. Therefore, the right amount to request is the
// maximum of those
// two numbers.
SHMEM_SZ = MAX(
sizeof(half) * (BLOCK_COL_TILES * M) * (CHUNK_K * K + SKEW_HALF) * 2,
M * (BLOCK_ROW_WARPS * WARP_ROW_TILES) * N *
(BLOCK_COL_WARPS * WARP_COL_TILES) * sizeof(float))
};
printf("Required shared memory size: %lu Kb\n", SHMEM_SZ / 1024UL);
const float alpha = 1.1f;
const float beta = 1.2f;
cudaEvent_t start, stop;
checkCudaErrors(cudaEventCreate(&start));
checkCudaErrors(cudaEventCreate(&stop));
checkCudaErrors(cudaEventRecord(start));
// If enough shared memory available on the GPU use high performant kernel
if (deviceProp.sharedMemPerMultiprocessor >= SHMEM_SZ) {
printf("Computing... using high performance kernel compute_gemm \n");
checkCudaErrors(cudaFuncSetAttribute(
compute_gemm, cudaFuncAttributeMaxDynamicSharedMemorySize, SHMEM_SZ));
checkKernelErrors(
(compute_gemm<<<deviceProp.multiProcessorCount, THREADS_PER_BLOCK,
SHMEM_SZ>>>(A, B, C, D, alpha, beta)));
#if CPU_DEBUG
checkCudaErrors(cudaMemcpy(result_hD, D,
sizeof(float) * M_GLOBAL * N_GLOBAL,
cudaMemcpyDeviceToHost));
#endif
} else {
dim3 gridDim;
dim3 blockDim;
// blockDim.x must be a multple of warpSize
// 128x4 means we have 16 warps and a block computes a 64x64 output tile
blockDim.x = 128;
blockDim.y = 4;
gridDim.x = (M_GLOBAL + (WMMA_M * blockDim.x / 32 - 1)) /
(WMMA_M * blockDim.x / 32);
gridDim.y = (N_GLOBAL + WMMA_N * blockDim.y - 1) / (WMMA_N * blockDim.y);
printf("Computing... using simple_wmma_gemm kernel\n");
simple_wmma_gemm<<<gridDim, blockDim>>>(A, B, C, D, M_GLOBAL, N_GLOBAL,
K_GLOBAL, alpha, beta);
#if CPU_DEBUG
checkCudaErrors(cudaMemcpy(result_hD, D,
sizeof(float) * M_GLOBAL * N_GLOBAL,
cudaMemcpyDeviceToHost));
#endif
}
checkCudaErrors(cudaEventRecord(stop));
checkCudaErrors(cudaEventSynchronize(stop));
#if CPU_DEBUG
printf("Verifying correctness of the computations...\n");
memcpy(result_host, C_h, sizeof(float) * M_GLOBAL * N_GLOBAL);
matMultiplyOnHost(A_h, B_h, result_host, alpha, beta, M_GLOBAL, K_GLOBAL,
K_GLOBAL, N_GLOBAL, M_GLOBAL, N_GLOBAL);
for (int i = 0; i < N_GLOBAL * M_GLOBAL; i++) {
if (fabs(result_hD[i] - result_host[i]) > 0.1f)
printf("mismatch i=%d result_hD=%f result_host=%f\n", i, result_hD[i],
result_host[i]);
}
free(result_hD);
free(result_host);
#endif
float milliseconds = 0;
checkCudaErrors(cudaEventElapsedTime(&milliseconds, start, stop));
printf("Time: %f ms\n", milliseconds);
printf("TFLOPS: %.2f\n", static_cast<double>((static_cast<double>(M_GLOBAL) *
N_GLOBAL * K_GLOBAL * 2) /
(milliseconds / 1000.)) /
1e12);
free(A_h);
free(B_h);
free(C_h);
checkCudaErrors(cudaFree(reinterpret_cast<void *>(A)));
checkCudaErrors(cudaFree(reinterpret_cast<void *>(B)));
checkCudaErrors(cudaFree(reinterpret_cast<void *>(C)));
checkCudaErrors(cudaFree(reinterpret_cast<void *>(D)));
return 0;
C[i * numCColumns + j] = temp * alpha + beta * C[i * numCColumns + j];
}
}
}
int main(int argc, char **argv)
{
printf("Initializing...\n");
int dev = findCudaDevice(argc, (const char **)argv);
cudaDeviceProp deviceProp;
checkCudaErrors(cudaGetDeviceProperties(&deviceProp, dev));
// Tensor cores require a GPU of Volta (SM7X) architecture or higher.
if (deviceProp.major < 7) {
printf("cudaTensorCoreGemm requires SM 7.0 or higher to use Tensor "
"Cores. Exiting...\n");
exit(EXIT_WAIVED);
}
printf("M: %d (%d x %d)\n", M_GLOBAL, M, M_TILES);
printf("N: %d (%d x %d)\n", N_GLOBAL, N, N_TILES);
printf("K: %d (%d x %d)\n", K_GLOBAL, K, K_TILES);
half *A_h = NULL;
half *B_h = NULL;
float *C_h = NULL;
#if CPU_DEBUG
float *result_hD = NULL;
float *result_host = NULL;
#endif
A_h = (half *)malloc(sizeof(half) * M_GLOBAL * K_GLOBAL);
B_h = (half *)malloc(sizeof(half) * K_GLOBAL * N_GLOBAL);
C_h = (float *)malloc(sizeof(float) * M_GLOBAL * N_GLOBAL);
#if CPU_DEBUG
result_hD = (float *)malloc(sizeof(float) * M_GLOBAL * N_GLOBAL);
result_host = (float *)malloc(sizeof(float) * M_GLOBAL * N_GLOBAL);
#endif
half *A = NULL;
half *B = NULL;
float *C = NULL;
float *D = NULL;
checkCudaErrors(cudaMalloc(reinterpret_cast<void **>(&A), sizeof(half) * M_GLOBAL * K_GLOBAL));
checkCudaErrors(cudaMalloc(reinterpret_cast<void **>(&B), sizeof(half) * N_GLOBAL * K_GLOBAL));
checkCudaErrors(cudaMalloc(reinterpret_cast<void **>(&C), sizeof(float) * M_GLOBAL * N_GLOBAL));
checkCudaErrors(cudaMalloc(reinterpret_cast<void **>(&D), sizeof(float) * M_GLOBAL * N_GLOBAL));
assert(((unsigned long long)A) % 128 == 0);
assert(((unsigned long long)B) % 128 == 0);
assert(((unsigned long long)C) % 128 == 0);
assert(((unsigned long long)D) % 128 == 0);
init_host_matrices(A_h, B_h, C_h);
printf("Preparing data for GPU...\n");
checkCudaErrors(cudaMemcpy(A, A_h, sizeof(half) * M_GLOBAL * K_GLOBAL, cudaMemcpyHostToDevice));
checkCudaErrors(cudaMemcpy(B, B_h, sizeof(half) * N_GLOBAL * K_GLOBAL, cudaMemcpyHostToDevice));
checkCudaErrors(cudaMemcpy(C, C_h, sizeof(float) * M_GLOBAL * N_GLOBAL, cudaMemcpyHostToDevice));
checkCudaErrors(cudaMemset(D, 0, sizeof(float) * M_GLOBAL * N_GLOBAL));
enum {
// Compute the right amount of shared memory to request.
// We need shared memory to hold per-CTA C and D matrix tiles, and to cache
// per-CTA chunks
// of the A and B matrices. Therefore, the right amount to request is the
// maximum of those
// two numbers.
SHMEM_SZ = MAX(sizeof(half) * (BLOCK_COL_TILES * M) * (CHUNK_K * K + SKEW_HALF) * 2,
M * (BLOCK_ROW_WARPS * WARP_ROW_TILES) * N * (BLOCK_COL_WARPS * WARP_COL_TILES) * sizeof(float))
};
printf("Required shared memory size: %lu Kb\n", SHMEM_SZ / 1024UL);
const float alpha = 1.1f;
const float beta = 1.2f;
cudaEvent_t start, stop;
checkCudaErrors(cudaEventCreate(&start));
checkCudaErrors(cudaEventCreate(&stop));
checkCudaErrors(cudaEventRecord(start));
// If enough shared memory available on the GPU use high performant kernel
if (deviceProp.sharedMemPerMultiprocessor >= SHMEM_SZ) {
printf("Computing... using high performance kernel compute_gemm \n");
checkCudaErrors(cudaFuncSetAttribute(compute_gemm, cudaFuncAttributeMaxDynamicSharedMemorySize, SHMEM_SZ));
checkKernelErrors(
(compute_gemm<<<deviceProp.multiProcessorCount, THREADS_PER_BLOCK, SHMEM_SZ>>>(A, B, C, D, alpha, beta)));
#if CPU_DEBUG
checkCudaErrors(cudaMemcpy(result_hD, D, sizeof(float) * M_GLOBAL * N_GLOBAL, cudaMemcpyDeviceToHost));
#endif
}
else {
dim3 gridDim;
dim3 blockDim;
// blockDim.x must be a multple of warpSize
// 128x4 means we have 16 warps and a block computes a 64x64 output tile
blockDim.x = 128;
blockDim.y = 4;
gridDim.x = (M_GLOBAL + (WMMA_M * blockDim.x / 32 - 1)) / (WMMA_M * blockDim.x / 32);
gridDim.y = (N_GLOBAL + WMMA_N * blockDim.y - 1) / (WMMA_N * blockDim.y);
printf("Computing... using simple_wmma_gemm kernel\n");
simple_wmma_gemm<<<gridDim, blockDim>>>(A, B, C, D, M_GLOBAL, N_GLOBAL, K_GLOBAL, alpha, beta);
#if CPU_DEBUG
checkCudaErrors(cudaMemcpy(result_hD, D, sizeof(float) * M_GLOBAL * N_GLOBAL, cudaMemcpyDeviceToHost));
#endif
}
checkCudaErrors(cudaEventRecord(stop));
checkCudaErrors(cudaEventSynchronize(stop));
#if CPU_DEBUG
printf("Verifying correctness of the computations...\n");
memcpy(result_host, C_h, sizeof(float) * M_GLOBAL * N_GLOBAL);
matMultiplyOnHost(A_h, B_h, result_host, alpha, beta, M_GLOBAL, K_GLOBAL, K_GLOBAL, N_GLOBAL, M_GLOBAL, N_GLOBAL);
for (int i = 0; i < N_GLOBAL * M_GLOBAL; i++) {
if (fabs(result_hD[i] - result_host[i]) > 0.1f)
printf("mismatch i=%d result_hD=%f result_host=%f\n", i, result_hD[i], result_host[i]);
}
free(result_hD);
free(result_host);
#endif
float milliseconds = 0;
checkCudaErrors(cudaEventElapsedTime(&milliseconds, start, stop));
printf("Time: %f ms\n", milliseconds);
printf("TFLOPS: %.2f\n",
static_cast<double>((static_cast<double>(M_GLOBAL) * N_GLOBAL * K_GLOBAL * 2) / (milliseconds / 1000.))
/ 1e12);
free(A_h);
free(B_h);
free(C_h);
checkCudaErrors(cudaFree(reinterpret_cast<void *>(A)));
checkCudaErrors(cudaFree(reinterpret_cast<void *>(B)));
checkCudaErrors(cudaFree(reinterpret_cast<void *>(C)));
checkCudaErrors(cudaFree(reinterpret_cast<void *>(D)));
return 0;
}

332
Samples/3_CUDA_Features/dmmaTensorCoreGemm/dmmaTensorCoreGemm.cu
View File

@@ -59,18 +59,18 @@
// but carefully enough to avoid local memory use.
#include <assert.h>
#include <stdio.h>
#include <cuda.h>
#include <mma.h>
#include <cooperative_groups.h>
#include <cooperative_groups/memcpy_async.h>
#include <cuda/std/type_traits>
#include <cuda.h>
#include <cuda/barrier>
#include <cuda/pipeline>
#include <cuda/std/type_traits>
#include <mma.h>
#include <stdio.h>
// helper functions and utilities to work with CUDA
#include <helper_functions.h>
#include <helper_cuda.h>
#include <helper_functions.h>
// Externally configurable parameters.
@@ -111,7 +111,7 @@
// Implementation constants.
#define WARPS_PER_BLOCK 8
#define WARPS_PER_BLOCK 8
#define THREADS_PER_BLOCK (WARP_SIZE * WARPS_PER_BLOCK)
#if SHARED_MEMORY_LIMIT_64K
@@ -127,10 +127,10 @@
#define CHUNK_K 16
#endif
#define CHUNK_LINE_BYTES (CHUNK_K * K * sizeof(double))
#define WARP_COPY_BYTES (WARP_SIZE * sizeof(int4))
#define CHUNK_LINE_BYTES (CHUNK_K * K * sizeof(double))
#define WARP_COPY_BYTES (WARP_SIZE * sizeof(int4))
#define CHUNK_COPY_LINES_PER_WARP (WARP_COPY_BYTES / CHUNK_LINE_BYTES)
#define CHUNK_COPY_LINE_LANES (WARP_SIZE / CHUNK_COPY_LINES_PER_WARP)
#define CHUNK_COPY_LINE_LANES (WARP_SIZE / CHUNK_COPY_LINES_PER_WARP)
#define BLOCK_ROW_WARPS 2
#define BLOCK_COL_WARPS 4
@@ -160,26 +160,28 @@
// we must keep each row and column 256-bit aligned, as required by nvcuda::wmma::load_matrix_sync.
#define SKEW_DOUBLE 4
#define checkKernelErrors(expr) do { \
expr; \
\
cudaError_t __err = cudaGetLastError(); \
if (__err != cudaSuccess) { \
printf("Line %d: '%s' failed: %s\n", __LINE__, # expr, cudaGetErrorString(__err)); \
abort(); \
} \
} while(0)
#define checkKernelErrors(expr) \
do { \
expr; \
\
cudaError_t __err = cudaGetLastError(); \
if (__err != cudaSuccess) { \
printf("Line %d: '%s' failed: %s\n", __LINE__, #expr, cudaGetErrorString(__err)); \
abort(); \
} \
} while (0)
enum kernels
{
dmma_shmem_gemm_async_copy = 0, // DMMA shmem using kernel with async_copy
dmma_shmem_gemm_cg_async_copy = 1, // DMMA shmem using kernel with cooperative groups async_copy
dmma_shmem_gemm = 2, // DMMA shmem using kernel normal copy (without async_copy).
simple_dmma_gemm = 3 // DMMA non-shmem using simple kernel.
enum kernels {
dmma_shmem_gemm_async_copy = 0, // DMMA shmem using kernel with async_copy
dmma_shmem_gemm_cg_async_copy = 1, // DMMA shmem using kernel with cooperative groups async_copy
dmma_shmem_gemm = 2, // DMMA shmem using kernel normal copy (without async_copy).
simple_dmma_gemm = 3 // DMMA non-shmem using simple kernel.
};
const char* kernelNames[] = {"compute_dgemm_async_copy", "compute_dgemm_cg_async_copy",
"compute_dgemm", "simple_wmma_gemm"};
const char *kernelNames[] = {"compute_dgemm_async_copy",
"compute_dgemm_cg_async_copy",
"compute_dgemm",
"simple_wmma_gemm"};
using namespace nvcuda;
namespace cg = cooperative_groups;
@@ -188,18 +190,18 @@ __host__ void init_host_matrices(double *a, double *b, double *c)
{
for (int i = 0; i < M_GLOBAL; i++) {
for (int j = 0; j < K_GLOBAL; j++) {
a[i*K_GLOBAL+j] = (double) (rand() % 3);
a[i * K_GLOBAL + j] = (double)(rand() % 3);
}
}
for (int i = 0; i < N_GLOBAL; i++) {
for (int j = 0; j < K_GLOBAL; j++) {
b[i*K_GLOBAL+j] = (double) (rand() % 3);
b[i * K_GLOBAL + j] = (double)(rand() % 3);
}
}
for (int t = 0; t < M_GLOBAL * N_GLOBAL; t++) {
c[t] = (double) (rand() % 3);
c[t] = (double)(rand() % 3);
}
}
@@ -217,10 +219,12 @@ __global__ void compute_dgemm(const double *A, const double *B, const double *C,
// This pointer is used to access the C and D matrix tiles this warp computes.
double *shmem_warp_tile_ptr = (double*)&shmem[0][0] + (warpId / BLOCK_ROW_WARPS) * SHMEM_STRIDE * N * BLOCK_ROW_WARPS + (warpId % BLOCK_ROW_WARPS) * SHMEM_OFFSET;
double *shmem_warp_tile_ptr = (double *)&shmem[0][0]
+ (warpId / BLOCK_ROW_WARPS) * SHMEM_STRIDE * N * BLOCK_ROW_WARPS
+ (warpId % BLOCK_ROW_WARPS) * SHMEM_OFFSET;
// This pointer is used to stream the C and D matrices block-wide tile to and from shared memory.
double *shmem_warp_stream_ptr = (double*)&shmem[0][0] + warpId * SHMEM_STRIDE * N;
double *shmem_warp_stream_ptr = (double *)&shmem[0][0] + warpId * SHMEM_STRIDE * N;
// Adjust the beta scaler, as it'll be multiplied by alpha at the end of
// each tile computation. Technically this is not generally correct (may result
@@ -230,7 +234,7 @@ __global__ void compute_dgemm(const double *A, const double *B, const double *C,
// Each CTA slides along the 64 x 64 tiles from the top left corner of the matrix to the
// right and down, and selects the next tile to compute. Once there's no such tile,
// all warps in this CTA exit.
for(unsigned int block_pos = blockIdx.x;; block_pos += gridDim.x) {
for (unsigned int block_pos = blockIdx.x;; block_pos += gridDim.x) {
const unsigned int block_tile_i = ((block_pos * BLOCK_ROW_TILES) / N_TILES) * (BLOCK_COL_TILES);
const unsigned int block_tile_j = (block_pos * BLOCK_COL_TILES) % N_TILES;
@@ -240,7 +244,7 @@ __global__ void compute_dgemm(const double *A, const double *B, const double *C,
}
// This warp's pointer to the C matrix data to copy memory from to shared memory.
const size_t gmem_idx = (block_tile_i + warpId) * M * GLOBAL_MEM_STRIDE + block_tile_j * N;
const size_t gmem_idx = (block_tile_i + warpId) * M * GLOBAL_MEM_STRIDE + block_tile_j * N;
const double *src_gmem_warp_stream_ptr = &C[gmem_idx];
// Stream multiple C tiles to shared memory.
@@ -271,7 +275,7 @@ __global__ void compute_dgemm(const double *A, const double *B, const double *C,
// Scale the C matrix.
#pragma unroll
for (int i = 0; i < WARP_COL_TILES; i++) {
for (int i = 0; i < WARP_COL_TILES; i++) {
#pragma unroll
for (int j = 0; j < WARP_ROW_TILES; j++) {
#pragma unroll
@@ -283,16 +287,19 @@ __global__ void compute_dgemm(const double *A, const double *B, const double *C,
// Select what warp copies what matrix to shared memory.
// Warps 0-3 copy the A matrix, warps 4-7 copy the B matrix.
const double *warp_ptr = (warpId < (WARPS_PER_BLOCK/2)) ? (&A[block_tile_i * M * K_GLOBAL] + M * K_GLOBAL * (warpId % (WARPS_PER_BLOCK/2)) * 2) :
(&B[block_tile_j * N * K_GLOBAL] + N * K_GLOBAL * (warpId % (WARPS_PER_BLOCK/2)) * 2);
const double *warp_ptr =
(warpId < (WARPS_PER_BLOCK / 2))
? (&A[block_tile_i * M * K_GLOBAL] + M * K_GLOBAL * (warpId % (WARPS_PER_BLOCK / 2)) * 2)
: (&B[block_tile_j * N * K_GLOBAL] + N * K_GLOBAL * (warpId % (WARPS_PER_BLOCK / 2)) * 2);
// Go through the global K dimension by a fixed step at a time.
#pragma unroll
for (int tile_k = 0; tile_k < K_TILES; tile_k += CHUNK_K) {
// Copy slices of the A and B matrices to shared memory.
// The first half of the warps in the CTA copy the A matrix, the rest copy the B matrix.
size_t shmem_idx = warpId < (WARPS_PER_BLOCK/2) ? (M * (warpId % (WARPS_PER_BLOCK/2)) * 2) :
(N * (warpId % (WARPS_PER_BLOCK/2)) * 2 + shmem_idx_b_off);
size_t shmem_idx = warpId < (WARPS_PER_BLOCK / 2)
? (M * (warpId % (WARPS_PER_BLOCK / 2)) * 2)
: (N * (warpId % (WARPS_PER_BLOCK / 2)) * 2 + shmem_idx_b_off);
// First half of the warp copies the first row / column of the matrix,
// the second half of the warp copies the next.
@@ -302,9 +309,10 @@ __global__ void compute_dgemm(const double *A, const double *B, const double *C,
shmem_idx += laneId / CHUNK_COPY_LINE_LANES;
#pragma unroll
for(int i = 0; i < ((WARP_SIZE/2) / CHUNK_COPY_LINES_PER_WARP); i++) {
// Copy 16 bytes at once in each lane.
*((int4*)&shmem[shmem_idx][0] + (laneId % CHUNK_COPY_LINE_LANES)) = *((int4*)lane_ptr + (laneId % CHUNK_COPY_LINE_LANES));
for (int i = 0; i < ((WARP_SIZE / 2) / CHUNK_COPY_LINES_PER_WARP); i++) {
// Copy 16 bytes at once in each lane.
*((int4 *)&shmem[shmem_idx][0] + (laneId % CHUNK_COPY_LINE_LANES)) =
*((int4 *)lane_ptr + (laneId % CHUNK_COPY_LINE_LANES));
// Advance the global memory pointer and the shared memory index.
lane_ptr = lane_ptr + K_GLOBAL * CHUNK_COPY_LINES_PER_WARP;
@@ -321,8 +329,8 @@ __global__ void compute_dgemm(const double *A, const double *B, const double *C,
#pragma unroll
for (int i = 0; i < WARP_COL_TILES; i++) {
size_t shmem_idx_a = (warpId/2) * M * 2 + (i * M);
const double *tile_ptr = &shmem[shmem_idx_a][k_step * K];
size_t shmem_idx_a = (warpId / 2) * M * 2 + (i * M);
const double *tile_ptr = &shmem[shmem_idx_a][k_step * K];
wmma::load_matrix_sync(a[i], tile_ptr, K * CHUNK_K + SKEW_DOUBLE);
@@ -331,11 +339,10 @@ __global__ void compute_dgemm(const double *A, const double *B, const double *C,
if (i == 0) {
// Load the B matrix fragment once, because it is going to be reused
// against the other A matrix fragments.
size_t shmem_idx_b = shmem_idx_b_off + (WARP_ROW_TILES * N) * (warpId%2) + (j * N);
const double *tile_ptr = &shmem[shmem_idx_b][k_step * K];
size_t shmem_idx_b = shmem_idx_b_off + (WARP_ROW_TILES * N) * (warpId % 2) + (j * N);
const double *tile_ptr = &shmem[shmem_idx_b][k_step * K];
wmma::load_matrix_sync(b[j], tile_ptr, K * CHUNK_K + SKEW_DOUBLE);
}
wmma::mma_sync(c[i][j], a[i], b[j], c[i][j]);
@@ -370,8 +377,8 @@ __global__ void compute_dgemm(const double *A, const double *B, const double *C,
#pragma unroll
for (int i = 0; i < N; i++) {
*((int4*)(dst_gmem_warp_stream_ptr + GLOBAL_MEM_STRIDE * i) + laneId) =
*((int4*)(shmem_warp_stream_ptr + SHMEM_STRIDE * i) + laneId);
*((int4 *)(dst_gmem_warp_stream_ptr + GLOBAL_MEM_STRIDE * i) + laneId) =
*((int4 *)(shmem_warp_stream_ptr + SHMEM_STRIDE * i) + laneId);
}
__syncthreads();
@@ -379,7 +386,8 @@ __global__ void compute_dgemm(const double *A, const double *B, const double *C,
#endif
}
__global__ void compute_dgemm_async_copy(const double *A, const double *B, const double *C, double *D, double alpha, double beta)
__global__ void
compute_dgemm_async_copy(const double *A, const double *B, const double *C, double *D, double alpha, double beta)
{
#if __CUDA_ARCH__ >= 800
extern __shared__ double shmem[][CHUNK_K * K + SKEW_DOUBLE];
@@ -392,7 +400,8 @@ __global__ void compute_dgemm_async_copy(const double *A, const double *B, const
constexpr size_t shmem_idx_b_off = BLOCK_COL_TILES * M;
// This pointer is used to access the C and D matrix tiles this warp computes.
double *shmem_warp_tile_ptr = &shmem[0][0] + (warpId/BLOCK_ROW_WARPS) * SHMEM_STRIDE * N * BLOCK_ROW_WARPS + (warpId % BLOCK_ROW_WARPS) * SHMEM_OFFSET;
double *shmem_warp_tile_ptr = &shmem[0][0] + (warpId / BLOCK_ROW_WARPS) * SHMEM_STRIDE * N * BLOCK_ROW_WARPS
+ (warpId % BLOCK_ROW_WARPS) * SHMEM_OFFSET;
// This pointer is used to stream the C and D matrices block-wide tile to and from shared memory.
double *shmem_warp_stream_ptr = &shmem[0][0] + warpId * SHMEM_STRIDE * N;
@@ -404,13 +413,13 @@ __global__ void compute_dgemm_async_copy(const double *A, const double *B, const
cuda::pipeline<cuda::thread_scope_thread> pipe = cuda::make_pipeline();
const auto shape2 = cuda::aligned_size_t<alignof(double2)>(sizeof(double2));
const auto shape2 = cuda::aligned_size_t<alignof(double2)>(sizeof(double2));
constexpr int loadStride = 1; // load 2 double, left-shift by 1.
// Each CTA slides along the 64 x 64 tiles from the top left corner of the matrix to the
// right and down, and selects the next tile to compute. Once there's no such tile,
// all warps in this CTA exit.
for(unsigned int block_pos = blockIdx.x;; block_pos += gridDim.x) {
for (unsigned int block_pos = blockIdx.x;; block_pos += gridDim.x) {
const unsigned int block_tile_i = ((block_pos * BLOCK_ROW_TILES) / N_TILES) * (BLOCK_COL_TILES);
const unsigned int block_tile_j = (block_pos * BLOCK_COL_TILES) % N_TILES;
@@ -420,7 +429,7 @@ __global__ void compute_dgemm_async_copy(const double *A, const double *B, const
}
// This warp's pointer to the C matrix data to copy memory from to shared memory.
const size_t gmem_idx = (block_tile_i + warpId) * M * GLOBAL_MEM_STRIDE + block_tile_j * N;
const size_t gmem_idx = (block_tile_i + warpId) * M * GLOBAL_MEM_STRIDE + block_tile_j * N;
const double *src_gmem_warp_stream_ptr = &C[gmem_idx];
// Stream multiple C tiles to shared memory.
@@ -428,8 +437,9 @@ __global__ void compute_dgemm_async_copy(const double *A, const double *B, const
for (int i = 0; i < N; i++) {
pipe.producer_acquire();
cuda::memcpy_async(&shmem_warp_stream_ptr[(SHMEM_STRIDE * i) + (laneId << loadStride)],
&src_gmem_warp_stream_ptr[(GLOBAL_MEM_STRIDE * i) + (laneId << loadStride)],
shape2, pipe);
&src_gmem_warp_stream_ptr[(GLOBAL_MEM_STRIDE * i) + (laneId << loadStride)],
shape2,
pipe);
pipe.producer_commit();
}
@@ -463,12 +473,14 @@ __global__ void compute_dgemm_async_copy(const double *A, const double *B, const
// Select what warp copies what matrix to shared memory.
// Warps 0-3 copy the A matrix, warps 4-7 copy the B matrix.
const double *warp_ptr = (warpId < (WARPS_PER_BLOCK/2)) ? (&A[block_tile_i * M * K_GLOBAL] + M * K_GLOBAL * (warpId % (WARPS_PER_BLOCK/2)) * 2) :
(&B[block_tile_j * N * K_GLOBAL] + N * K_GLOBAL * (warpId % (WARPS_PER_BLOCK/2)) * 2);
const double *warp_ptr =
(warpId < (WARPS_PER_BLOCK / 2))
? (&A[block_tile_i * M * K_GLOBAL] + M * K_GLOBAL * (warpId % (WARPS_PER_BLOCK / 2)) * 2)
: (&B[block_tile_j * N * K_GLOBAL] + N * K_GLOBAL * (warpId % (WARPS_PER_BLOCK / 2)) * 2);
const int stridePerLaneCopy = (laneId / CHUNK_COPY_LINE_LANES);
constexpr int chunksPerLane = ((WARP_SIZE/2) / CHUNK_COPY_LINES_PER_WARP);
const int laneLoadElem = (laneId % CHUNK_COPY_LINE_LANES) << loadStride;
const int stridePerLaneCopy = (laneId / CHUNK_COPY_LINE_LANES);
constexpr int chunksPerLane = ((WARP_SIZE / 2) / CHUNK_COPY_LINES_PER_WARP);
const int laneLoadElem = (laneId % CHUNK_COPY_LINE_LANES) << loadStride;
// Go through the global K dimension by a fixed step at a time.
#pragma unroll
@@ -476,7 +488,8 @@ __global__ void compute_dgemm_async_copy(const double *A, const double *B, const
// Copy slices of the A and B matrices to shared memory.
// The first half of the warps in the CTA copy the A matrix, the rest copy the B matrix.
// As for DMMA M == N we use M for warp 4-7 + shmem_idx_b_off.
size_t shmem_idx = (M * (warpId % (WARPS_PER_BLOCK/2)) * 2) + (shmem_idx_b_off * (warpId/(WARPS_PER_BLOCK/2)));
size_t shmem_idx =
(M * (warpId % (WARPS_PER_BLOCK / 2)) * 2) + (shmem_idx_b_off * (warpId / (WARPS_PER_BLOCK / 2)));
// First half of the warp copies the first row / column of the matrix,
// the second half of the warp copies the next.
@@ -485,8 +498,8 @@ __global__ void compute_dgemm_async_copy(const double *A, const double *B, const
// Shift the second half of the warp to the next row / column in the shared memory.
shmem_idx += stridePerLaneCopy;
#pragma unroll
for(int i = 0; i < chunksPerLane; i++) {
// Copy 16 bytes at once in each lane.
for (int i = 0; i < chunksPerLane; i++) {
// Copy 16 bytes at once in each lane.
pipe.producer_acquire();
cuda::memcpy_async(&shmem[shmem_idx][laneLoadElem], lane_ptr, shape2, pipe);
@@ -508,8 +521,8 @@ __global__ void compute_dgemm_async_copy(const double *A, const double *B, const
wmma::fragment<wmma::matrix_b, M, N, K, double, wmma::col_major> b[WARP_ROW_TILES];
#pragma unroll
for (int i = 0; i < WARP_COL_TILES; i++) {
size_t shmem_idx_a = (warpId/2) * M * 2 + (i * M);
const double *tile_ptr = &shmem[shmem_idx_a][k_step * K];
size_t shmem_idx_a = (warpId / 2) * M * 2 + (i * M);
const double *tile_ptr = &shmem[shmem_idx_a][k_step * K];
wmma::load_matrix_sync(a[i], tile_ptr, K * CHUNK_K + SKEW_DOUBLE);
#pragma unroll
@@ -517,8 +530,8 @@ __global__ void compute_dgemm_async_copy(const double *A, const double *B, const
if (i == 0) {
// Load the B matrix fragment once, because it is going to be reused
// against the other A matrix fragments.
size_t shmem_idx_b = shmem_idx_b_off + (WARP_ROW_TILES * N) * (warpId%2) + (j * N);
const double *tile_ptr = &shmem[shmem_idx_b][k_step * K];
size_t shmem_idx_b = shmem_idx_b_off + (WARP_ROW_TILES * N) * (warpId % 2) + (j * N);
const double *tile_ptr = &shmem[shmem_idx_b][k_step * K];
wmma::load_matrix_sync(b[j], tile_ptr, K * CHUNK_K + SKEW_DOUBLE);
}
@@ -554,8 +567,8 @@ __global__ void compute_dgemm_async_copy(const double *A, const double *B, const
#pragma unroll
for (int i = 0; i < N; i++) {
*((int4*)(dst_gmem_warp_stream_ptr + GLOBAL_MEM_STRIDE * i) + laneId) =
*((int4*)(shmem_warp_stream_ptr + SHMEM_STRIDE * i) + laneId);
*((int4 *)(dst_gmem_warp_stream_ptr + GLOBAL_MEM_STRIDE * i) + laneId) =
*((int4 *)(shmem_warp_stream_ptr + SHMEM_STRIDE * i) + laneId);
}
__syncthreads();
@@ -563,15 +576,16 @@ __global__ void compute_dgemm_async_copy(const double *A, const double *B, const
#endif
}
__global__ void compute_dgemm_cg_async_copy(const double *A, const double *B, const double *C, double *D, double alpha, double beta)
__global__ void
compute_dgemm_cg_async_copy(const double *A, const double *B, const double *C, double *D, double alpha, double beta)
{
#if __CUDA_ARCH__ >= 800
extern __shared__ double shmem[][CHUNK_K * K + SKEW_DOUBLE];
auto cta = cg::this_thread_block();
auto tile32 = cg::tiled_partition<32>(cta);
auto cta = cg::this_thread_block();
auto tile32 = cg::tiled_partition<32>(cta);
constexpr int tileChunkCopySize = WARP_SIZE / CHUNK_COPY_LINES_PER_WARP;
auto tileChunkCopy = cg::tiled_partition<tileChunkCopySize>(cta);
auto tileChunkCopy = cg::tiled_partition<tileChunkCopySize>(cta);
// Warp and lane identification.
const unsigned int warpId = threadIdx.x / WARP_SIZE;
@@ -581,10 +595,11 @@ __global__ void compute_dgemm_async_copy(const double *A, const double *B, const
constexpr size_t shmem_idx_b_off = BLOCK_COL_TILES * M;
// This pointer is used to access the C and D matrix tiles this warp computes.
double *shmem_warp_tile_ptr = (double*)&shmem[0][0] + (warpId/2) * SHMEM_STRIDE * N * 2 + (warpId%2) * SHMEM_OFFSET;
double *shmem_warp_tile_ptr =
(double *)&shmem[0][0] + (warpId / 2) * SHMEM_STRIDE * N * 2 + (warpId % 2) * SHMEM_OFFSET;
// This pointer is used to stream the C and D matrices block-wide tile to and from shared memory.
double *shmem_warp_stream_ptr = (double*)&shmem[0][0] + warpId * SHMEM_STRIDE * N;
double *shmem_warp_stream_ptr = (double *)&shmem[0][0] + warpId * SHMEM_STRIDE * N;
// Adjust the beta scaler, as it'll be multiplied by alpha at the end of
// each tile computation. Technically this is not generally correct (may result
@@ -594,7 +609,7 @@ __global__ void compute_dgemm_async_copy(const double *A, const double *B, const
// Each CTA slides along the 64 x 64 tiles from the top left corner of the matrix to the
// right and down, and selects the next tile to compute. Once there's no such tile,
// all warps in this CTA exit.
for(unsigned int block_pos = blockIdx.x;; block_pos += gridDim.x) {
for (unsigned int block_pos = blockIdx.x;; block_pos += gridDim.x) {
const unsigned int block_tile_i = ((block_pos * BLOCK_ROW_TILES) / N_TILES) * (BLOCK_COL_TILES);
const unsigned int block_tile_j = (block_pos * BLOCK_COL_TILES) % N_TILES;
@@ -604,7 +619,7 @@ __global__ void compute_dgemm_async_copy(const double *A, const double *B, const
}
// This warp's pointer to the C matrix data to copy memory from to shared memory.
const size_t gmem_idx = (block_tile_i + warpId) * M * GLOBAL_MEM_STRIDE + block_tile_j * N;
const size_t gmem_idx = (block_tile_i + warpId) * M * GLOBAL_MEM_STRIDE + block_tile_j * N;
const double *src_gmem_warp_stream_ptr = &C[gmem_idx];
// Stream multiple C tiles to shared memory.
@@ -612,7 +627,8 @@ __global__ void compute_dgemm_async_copy(const double *A, const double *B, const
for (int i = 0; i < N; i++) {
auto dst_ptr = &shmem_warp_stream_ptr[(SHMEM_STRIDE * i)];
auto src_ptr = &src_gmem_warp_stream_ptr[(GLOBAL_MEM_STRIDE * i)];
cg::memcpy_async(tile32, dst_ptr, src_ptr, cuda::aligned_size_t<alignof(double2)>{tile32.size() * sizeof(double2)});
cg::memcpy_async(
tile32, dst_ptr, src_ptr, cuda::aligned_size_t<alignof(double2)>{tile32.size() * sizeof(double2)});
}
cg::wait(cta);
@@ -647,18 +663,20 @@ __global__ void compute_dgemm_async_copy(const double *A, const double *B, const
cg::wait(cta);
// Select what warp copies what matrix to shared memory.
// Warps 0-3 copy the A matrix, warps 4-7 copy the B matrix.
const double *warp_ptr = (warpId < 4) ? (&A[block_tile_i * M * K_GLOBAL] + M * K_GLOBAL * (warpId % (WARPS_PER_BLOCK/2)) * 2) :
(&B[block_tile_j * N * K_GLOBAL] + N * K_GLOBAL * (warpId % (WARPS_PER_BLOCK/2)) * 2);
const double *warp_ptr =
(warpId < 4) ? (&A[block_tile_i * M * K_GLOBAL] + M * K_GLOBAL * (warpId % (WARPS_PER_BLOCK / 2)) * 2)
: (&B[block_tile_j * N * K_GLOBAL] + N * K_GLOBAL * (warpId % (WARPS_PER_BLOCK / 2)) * 2);
const int stridePerLaneCopy = (laneId / CHUNK_COPY_LINE_LANES);
constexpr int chunksPerLane = ((WARP_SIZE/2) / CHUNK_COPY_LINES_PER_WARP);
const int stridePerLaneCopy = (laneId / CHUNK_COPY_LINE_LANES);
constexpr int chunksPerLane = ((WARP_SIZE / 2) / CHUNK_COPY_LINES_PER_WARP);
// Go through the global K dimension by a fixed step at a time.
#pragma unroll
for (int tile_k = 0; tile_k < K_TILES; tile_k += CHUNK_K) {
// Copy slices of the A and B matrices to shared memory.
// The first half of the warps in the CTA copy the A matrix, the rest copy the B matrix.
// As for DMMA M == N we use M for warp 4-7 + shmem_idx_b_off.
size_t shmem_idx = (M * (warpId % (WARPS_PER_BLOCK/2)) * 2) + (shmem_idx_b_off * (warpId/(WARPS_PER_BLOCK/2)));
size_t shmem_idx =
(M * (warpId % (WARPS_PER_BLOCK / 2)) * 2) + (shmem_idx_b_off * (warpId / (WARPS_PER_BLOCK / 2)));
// First half of the warp copies the first row / column of the matrix,
// the second half of the warp copies the next.
@@ -668,13 +686,15 @@ __global__ void compute_dgemm_async_copy(const double *A, const double *B, const
shmem_idx += stridePerLaneCopy;
#pragma unroll
for(int i = 0; i < chunksPerLane; i++) {
for (int i = 0; i < chunksPerLane; i++) {
// Copy 16 bytes at once in each lane.
auto dst_ptr = &shmem[shmem_idx][0];
auto src_ptr = lane_ptr;
cg::memcpy_async(tileChunkCopy, dst_ptr, src_ptr,
cuda::aligned_size_t<alignof(double2)>{tileChunkCopySize * sizeof(double2)});
cg::memcpy_async(tileChunkCopy,
dst_ptr,
src_ptr,
cuda::aligned_size_t<alignof(double2)>{tileChunkCopySize * sizeof(double2)});
// Advance the global memory pointer and the shared memory index.
lane_ptr = lane_ptr + K_GLOBAL * CHUNK_COPY_LINES_PER_WARP;
@@ -690,8 +710,8 @@ __global__ void compute_dgemm_async_copy(const double *A, const double *B, const
#pragma unroll
for (int i = 0; i < WARP_COL_TILES; i++) {
size_t shmem_idx_a = (warpId/2) * M * 2 + (i * M);
const double *tile_ptr = &shmem[shmem_idx_a][k_step * K];
size_t shmem_idx_a = (warpId / 2) * M * 2 + (i * M);
const double *tile_ptr = &shmem[shmem_idx_a][k_step * K];
wmma::load_matrix_sync(a[i], tile_ptr, K * CHUNK_K + SKEW_DOUBLE);
@@ -700,11 +720,10 @@ __global__ void compute_dgemm_async_copy(const double *A, const double *B, const
if (i == 0) {
// Load the B matrix fragment once, because it is going to be reused
// against the other A matrix fragments.
size_t shmem_idx_b = shmem_idx_b_off + (WARP_ROW_TILES * N) * (warpId%2) + (j * N);
const double *tile_ptr = &shmem[shmem_idx_b][k_step * K];
size_t shmem_idx_b = shmem_idx_b_off + (WARP_ROW_TILES * N) * (warpId % 2) + (j * N);
const double *tile_ptr = &shmem[shmem_idx_b][k_step * K];
wmma::load_matrix_sync(b[j], tile_ptr, K * CHUNK_K + SKEW_DOUBLE);
}
wmma::mma_sync(c[i][j], a[i], b[j], c[i][j]);
@@ -738,8 +757,8 @@ __global__ void compute_dgemm_async_copy(const double *A, const double *B, const
#pragma unroll
for (int i = 0; i < N; i++) {
*((int4*)(dst_gmem_warp_stream_ptr + GLOBAL_MEM_STRIDE * i) + laneId) =
*((int4*)(shmem_warp_stream_ptr + SHMEM_STRIDE * i) + laneId);
*((int4 *)(dst_gmem_warp_stream_ptr + GLOBAL_MEM_STRIDE * i) + laneId) =
*((int4 *)(shmem_warp_stream_ptr + SHMEM_STRIDE * i) + laneId);
}
cg::sync(cta);
}
@@ -748,12 +767,13 @@ __global__ void compute_dgemm_async_copy(const double *A, const double *B, const
// Performs an MxNxK DGEMM (C=alpha*A*B + beta*C) assuming:
// 1) Matrices are packed in memory.
// 2) M, N and K are multiples of 8, 8 and 4 respectively.
// 2) M, N and K are multiples of 8, 8 and 4 respectively.
// 3) A is row major, B is column major matrix.
// Note: This is a less performant version of the compute_dgemm kernel. It is designed for
// demonstration purposes only to show the CUDA WMMA API use without relying on
// availability of the shared memory.
__global__ void simple_wmma_gemm(double *a, double *b, double *c, double *d, int m_ld, int n_ld, int k_ld, double alpha, double beta)
__global__ void
simple_wmma_gemm(double *a, double *b, double *c, double *d, int m_ld, int n_ld, int k_ld, double alpha, double beta)
{
#if __CUDA_ARCH__ >= 800
// Leading dimensions. Packed with no transpositions.
@@ -768,8 +788,8 @@ __global__ void simple_wmma_gemm(double *a, double *b, double *c, double *d, int
// Declare the fragments
wmma::fragment<wmma::matrix_a, M, N, K, double, wmma::row_major> a_frag;
wmma::fragment<wmma::matrix_b, M, N, K, double, wmma::col_major> b_frag;
wmma::fragment<wmma::accumulator, M, N, K, double> acc_frag;
wmma::fragment<wmma::accumulator, M, N, K, double> c_frag;
wmma::fragment<wmma::accumulator, M, N, K, double> acc_frag;
wmma::fragment<wmma::accumulator, M, N, K, double> c_frag;
wmma::fill_fragment(acc_frag, 0.0f);
@@ -799,7 +819,7 @@ __global__ void simple_wmma_gemm(double *a, double *b, double *c, double *d, int
if (cRow < m_ld && cCol < n_ld) {
wmma::load_matrix_sync(c_frag, c + cCol + cRow * ldc, ldc, wmma::mem_row_major);
for(int i=0; i < c_frag.num_elements; i++) {
for (int i = 0; i < c_frag.num_elements; i++) {
c_frag.x[i] = alpha * acc_frag.x[i] + beta * c_frag.x[i];
}
@@ -809,11 +829,17 @@ __global__ void simple_wmma_gemm(double *a, double *b, double *c, double *d, int
#endif
}
__host__ void matMultiplyOnHost(double *A, double *B, double *C,
float alpha, float beta,
int numARows, int numAColumns,
int numBRows, int numBColumns,
int numCRows, int numCColumns)
__host__ void matMultiplyOnHost(double *A,
double *B,
double *C,
float alpha,
float beta,
int numARows,
int numAColumns,
int numBRows,
int numBColumns,
int numCRows,
int numCColumns)
{
for (int i = 0; i < numCRows; i++) {
for (int j = 0; j < numCColumns; j++) {
@@ -824,7 +850,7 @@ __host__ void matMultiplyOnHost(double *A, double *B, double *C,
temp += A[i * numAColumns + k] * B[j * numBRows + k];
}
C[i*numCColumns + j] = temp * alpha + beta * C[i * numCColumns + j];
C[i * numCColumns + j] = temp * alpha + beta * C[i * numCColumns + j];
}
}
}
@@ -852,16 +878,16 @@ int main(int argc, char **argv)
double *B_h = NULL;
double *C_h = NULL;
#if CPU_DEBUG
double *result_hD = NULL;
double *result_hD = NULL;
double *result_host = NULL;
#endif
A_h = (double*) malloc(sizeof(double) * M_GLOBAL * K_GLOBAL);
B_h = (double*) malloc(sizeof(double) * K_GLOBAL * N_GLOBAL);
C_h = (double*) malloc(sizeof(double) * M_GLOBAL * N_GLOBAL);
A_h = (double *)malloc(sizeof(double) * M_GLOBAL * K_GLOBAL);
B_h = (double *)malloc(sizeof(double) * K_GLOBAL * N_GLOBAL);
C_h = (double *)malloc(sizeof(double) * M_GLOBAL * N_GLOBAL);
#if CPU_DEBUG
result_hD = (double*) malloc(sizeof(double) * M_GLOBAL * N_GLOBAL);
result_host = (double*) malloc(sizeof(double) * M_GLOBAL * N_GLOBAL);
result_hD = (double *)malloc(sizeof(double) * M_GLOBAL * N_GLOBAL);
result_host = (double *)malloc(sizeof(double) * M_GLOBAL * N_GLOBAL);
#endif
double *A = NULL;
@@ -869,10 +895,10 @@ int main(int argc, char **argv)
double *C = NULL;
double *D = NULL;
checkCudaErrors(cudaMalloc((void**)&A, sizeof(double) * M_GLOBAL * K_GLOBAL));
checkCudaErrors(cudaMalloc((void**)&B, sizeof(double) * N_GLOBAL * K_GLOBAL));
checkCudaErrors(cudaMalloc((void**)&C, sizeof(double) * M_GLOBAL * N_GLOBAL));
checkCudaErrors(cudaMalloc((void**)&D, sizeof(double) * M_GLOBAL * N_GLOBAL));
checkCudaErrors(cudaMalloc((void **)&A, sizeof(double) * M_GLOBAL * K_GLOBAL));
checkCudaErrors(cudaMalloc((void **)&B, sizeof(double) * N_GLOBAL * K_GLOBAL));
checkCudaErrors(cudaMalloc((void **)&C, sizeof(double) * M_GLOBAL * N_GLOBAL));
checkCudaErrors(cudaMalloc((void **)&D, sizeof(double) * M_GLOBAL * N_GLOBAL));
assert(((unsigned long long)A) % 128 == 0);
assert(((unsigned long long)B) % 128 == 0);
@@ -900,7 +926,7 @@ int main(int argc, char **argv)
printf("Required shared memory size: %lu Kb\n", SHMEM_SZ / 1024UL);
const double alpha = 1.1f;
const double beta = 1.2f;
const double beta = 1.2f;
cudaEvent_t start, stop;
@@ -913,45 +939,47 @@ int main(int argc, char **argv)
// kernel to run - default (dmma_shmem_gemm_async_copy == 0)
if (checkCmdLineFlag(argc, (const char **)argv, "kernel")) {
int kernel_number = getCmdLineArgumentInt(argc, (const char **)argv, "kernel");
if (kernel_number < 4)
{
if (kernel_number < 4) {
selected_kernel = (kernels)kernel_number;
}
else
{
else {
printf("Error: kernel number should be between 0 to 3, you have entered %d\n", kernel_number);
exit(EXIT_FAILURE);
}
}
// If enough shared memory available on the GPU use high performant kernel
if ((deviceProp.sharedMemPerMultiprocessor >= SHMEM_SZ) && (selected_kernel != simple_dmma_gemm))
{
if ((deviceProp.sharedMemPerMultiprocessor >= SHMEM_SZ) && (selected_kernel != simple_dmma_gemm)) {
printf("Computing using high performance kernel = %d - %s\n", selected_kernel, kernelNames[selected_kernel]);
switch (selected_kernel)
{
case dmma_shmem_gemm_async_copy :
default:
checkCudaErrors(cudaFuncSetAttribute(compute_dgemm_async_copy, cudaFuncAttributeMaxDynamicSharedMemorySize, SHMEM_SZ));
checkKernelErrors((compute_dgemm_async_copy<<<deviceProp.multiProcessorCount*3, THREADS_PER_BLOCK, SHMEM_SZ>>>(A, B, C, D, alpha, beta)));
break;
case dmma_shmem_gemm_cg_async_copy :
checkCudaErrors(cudaFuncSetAttribute(compute_dgemm_cg_async_copy, cudaFuncAttributeMaxDynamicSharedMemorySize, SHMEM_SZ));
checkKernelErrors((compute_dgemm_cg_async_copy<<<deviceProp.multiProcessorCount*3, THREADS_PER_BLOCK, SHMEM_SZ>>>(A, B, C, D, alpha, beta)));
break;
case dmma_shmem_gemm :
checkCudaErrors(cudaFuncSetAttribute(compute_dgemm, cudaFuncAttributeMaxDynamicSharedMemorySize, SHMEM_SZ));
checkKernelErrors((compute_dgemm<<<deviceProp.multiProcessorCount*2, THREADS_PER_BLOCK, SHMEM_SZ>>>(A, B, C, D, alpha, beta)));
break;
switch (selected_kernel) {
case dmma_shmem_gemm_async_copy:
default:
checkCudaErrors(
cudaFuncSetAttribute(compute_dgemm_async_copy, cudaFuncAttributeMaxDynamicSharedMemorySize, SHMEM_SZ));
checkKernelErrors(
(compute_dgemm_async_copy<<<deviceProp.multiProcessorCount * 3, THREADS_PER_BLOCK, SHMEM_SZ>>>(
A, B, C, D, alpha, beta)));
break;
case dmma_shmem_gemm_cg_async_copy:
checkCudaErrors(cudaFuncSetAttribute(
compute_dgemm_cg_async_copy, cudaFuncAttributeMaxDynamicSharedMemorySize, SHMEM_SZ));
checkKernelErrors(
(compute_dgemm_cg_async_copy<<<deviceProp.multiProcessorCount * 3, THREADS_PER_BLOCK, SHMEM_SZ>>>(
A, B, C, D, alpha, beta)));
break;
case dmma_shmem_gemm:
checkCudaErrors(cudaFuncSetAttribute(compute_dgemm, cudaFuncAttributeMaxDynamicSharedMemorySize, SHMEM_SZ));
checkKernelErrors((compute_dgemm<<<deviceProp.multiProcessorCount * 2, THREADS_PER_BLOCK, SHMEM_SZ>>>(
A, B, C, D, alpha, beta)));
break;
}
#if CPU_DEBUG
checkCudaErrors(cudaMemcpy(result_hD, D, sizeof(double)*M_GLOBAL*N_GLOBAL, cudaMemcpyDeviceToHost));
checkCudaErrors(cudaMemcpy(result_hD, D, sizeof(double) * M_GLOBAL * N_GLOBAL, cudaMemcpyDeviceToHost));
#endif
}
else
{
else {
dim3 gridDim;
dim3 blockDim;
@@ -978,25 +1006,19 @@ int main(int argc, char **argv)
memcpy(result_host, C_h, sizeof(double) * M_GLOBAL * N_GLOBAL);
matMultiplyOnHost(A_h, B_h, result_host,
alpha, beta,
M_GLOBAL, K_GLOBAL,
K_GLOBAL, N_GLOBAL,
M_GLOBAL, N_GLOBAL);
matMultiplyOnHost(A_h, B_h, result_host, alpha, beta, M_GLOBAL, K_GLOBAL, K_GLOBAL, N_GLOBAL, M_GLOBAL, N_GLOBAL);
size_t number_of_matches = 0;
for (int i = 0; i < N_GLOBAL*M_GLOBAL; i++) {
if (fabs(result_hD[i] - result_host[i]) > 0.1f)
{
for (int i = 0; i < N_GLOBAL * M_GLOBAL; i++) {
if (fabs(result_hD[i] - result_host[i]) > 0.1f) {
printf("mismatch i=%d result_hD=%f result_host=%f\n", i, result_hD[i], result_host[i]);
break;
}
else
{
else {
number_of_matches++;
}
}
printf("number_of_matches = %zu out of = %d \n", number_of_matches, N_GLOBAL*M_GLOBAL);
printf("number_of_matches = %zu out of = %d \n", number_of_matches, N_GLOBAL * M_GLOBAL);
free(result_hD);
free(result_host);
#endif
@@ -1006,15 +1028,15 @@ int main(int argc, char **argv)
checkCudaErrors(cudaEventElapsedTime(&milliseconds, start, stop));
printf("Time: %f ms\n", milliseconds);
printf("FP64 TFLOPS: %.2f\n", (((double)M_GLOBAL * N_GLOBAL * K_GLOBAL * 2)/(milliseconds/1000.)) / 1e12);
printf("FP64 TFLOPS: %.2f\n", (((double)M_GLOBAL * N_GLOBAL * K_GLOBAL * 2) / (milliseconds / 1000.)) / 1e12);
free(A_h);
free(B_h);
free(C_h);
checkCudaErrors(cudaFree((void*)A));
checkCudaErrors(cudaFree((void*)B));
checkCudaErrors(cudaFree((void*)C));
checkCudaErrors(cudaFree((void*)D));
checkCudaErrors(cudaFree((void *)A));
checkCudaErrors(cudaFree((void *)B));
checkCudaErrors(cudaFree((void *)C));
checkCudaErrors(cudaFree((void *)D));
return 0;
}

1483
Samples/3_CUDA_Features/globalToShmemAsyncCopy/globalToShmemAsyncCopy.cu
View File

File diff suppressed because it is too large Load Diff

135
Samples/3_CUDA_Features/graphConditionalNodes/graphConditionalNodes.cu
View File

@@ -66,15 +66,12 @@ __global__ void ifGraphKernelA(char *dPtr, cudaGraphConditionalHandle handle)
}
// This kernel will only be executed if the condition is true
__global__ void ifGraphKernelC(void)
{
printf("GPU: Hello from the GPU! The condition was true.\n");
}
__global__ void ifGraphKernelC(void) { printf("GPU: Hello from the GPU! The condition was true.\n"); }
// Setup and launch the graph
void simpleIfGraph(void)
{
cudaGraph_t graph;
cudaGraph_t graph;
cudaGraphExec_t graphExec;
cudaGraphNode_t kernelNode;
cudaGraphNode_t conditionalNode;
@@ -94,25 +91,25 @@ void simpleIfGraph(void)
// Use a kernel upstream of the conditional to set the handle value
cudaGraphNodeParams params = {cudaGraphNodeTypeKernel};
params.kernel.func = (void *)ifGraphKernelA;
params.kernel.func = (void *)ifGraphKernelA;
params.kernel.blockDim.x = params.kernel.blockDim.y = params.kernel.blockDim.z = 1;
params.kernel.gridDim.x = params.kernel.gridDim.y = params.kernel.gridDim.z = 1;
params.kernel.kernelParams = kernelArgs;
kernelArgs[0] = &dPtr;
kernelArgs[1] = &handle;
params.kernel.kernelParams = kernelArgs;
kernelArgs[0] = &dPtr;
kernelArgs[1] = &handle;
checkCudaErrors(cudaGraphAddNode(&kernelNode, graph, NULL, 0, &params));
cudaGraphNodeParams cParams = {cudaGraphNodeTypeConditional};
cParams.conditional.handle = handle;
cParams.conditional.type = cudaGraphCondTypeIf;
cParams.conditional.size = 1;
cParams.conditional.handle = handle;
cParams.conditional.type = cudaGraphCondTypeIf;
cParams.conditional.size = 1;
checkCudaErrors(cudaGraphAddNode(&conditionalNode, graph, &kernelNode, 1, &cParams));
cudaGraph_t bodyGraph = cParams.conditional.phGraph_out[0];
// Populate the body of the conditional node
cudaGraphNode_t bodyNode;
params.kernel.func = (void *)ifGraphKernelC;
params.kernel.func = (void *)ifGraphKernelC;
params.kernel.kernelParams = nullptr;
checkCudaErrors(cudaGraphAddNode(&bodyNode, bodyGraph, NULL, 0, &params));
@@ -159,8 +156,7 @@ __global__ void doWhileEmptyKernel(void)
__global__ void doWhileLoopKernel(char *dPtr, cudaGraphConditionalHandle handle)
{
if (--(*dPtr) == 0)
{
if (--(*dPtr) == 0) {
cudaGraphSetConditional(handle, 0);
}
printf("GPU: counter = %d\n", *dPtr);
@@ -168,7 +164,7 @@ __global__ void doWhileLoopKernel(char *dPtr, cudaGraphConditionalHandle handle)
void simpleDoWhileGraph(void)
{
cudaGraph_t graph;
cudaGraph_t graph;
cudaGraphExec_t graphExec;
cudaGraphNode_t conditionalNode;
@@ -183,9 +179,9 @@ void simpleDoWhileGraph(void)
checkCudaErrors(cudaGraphConditionalHandleCreate(&handle, graph, 1, cudaGraphCondAssignDefault));
cudaGraphNodeParams cParams = {cudaGraphNodeTypeConditional};
cParams.conditional.handle = handle;
cParams.conditional.type = cudaGraphCondTypeWhile;
cParams.conditional.size = 1;
cParams.conditional.handle = handle;
cParams.conditional.type = cudaGraphCondTypeWhile;
cParams.conditional.size = 1;
checkCudaErrors(cudaGraphAddNode(&conditionalNode, graph, NULL, 0, &cParams));
cudaGraph_t bodyGraph = cParams.conditional.phGraph_out[0];
@@ -193,7 +189,8 @@ void simpleDoWhileGraph(void)
cudaStream_t captureStream;
checkCudaErrors(cudaStreamCreate(&captureStream));
checkCudaErrors(cudaStreamBeginCaptureToGraph(captureStream, bodyGraph, nullptr, nullptr, 0, cudaStreamCaptureModeGlobal));
checkCudaErrors(
cudaStreamBeginCaptureToGraph(captureStream, bodyGraph, nullptr, nullptr, 0, cudaStreamCaptureModeGlobal));
doWhileEmptyKernel<<<1, 1, 0, captureStream>>>();
doWhileEmptyKernel<<<1, 1, 0, captureStream>>>();
doWhileLoopKernel<<<1, 1, 0, captureStream>>>(dPtr, handle);
@@ -238,8 +235,7 @@ void simpleDoWhileGraph(void)
__global__ void capturedWhileKernel(char *dPtr, cudaGraphConditionalHandle handle)
{
printf("GPU: counter = %d\n", *dPtr);
if (*dPtr)
{
if (*dPtr) {
(*dPtr)--;
}
cudaGraphSetConditional(handle, *dPtr);
@@ -253,12 +249,12 @@ __global__ void capturedWhileEmptyKernel(void)
void capturedWhileGraph(void)
{
cudaGraph_t graph;
cudaGraph_t graph;
cudaGraphExec_t graphExec;
cudaStreamCaptureStatus status;
const cudaGraphNode_t *dependencies;
size_t numDependencies;
const cudaGraphNode_t *dependencies;
size_t numDependencies;
// Allocate a byte of device memory to use as input
char *dPtr;
@@ -284,17 +280,18 @@ void capturedWhileGraph(void)
checkCudaErrors(cudaStreamGetCaptureInfo(captureStream, &status, NULL, &graph, &dependencies, &numDependencies));
// Insert conditional node B
cudaGraphNode_t conditionalNode;
cudaGraphNode_t conditionalNode;
cudaGraphNodeParams cParams = {cudaGraphNodeTypeConditional};
cParams.conditional.handle = handle;
cParams.conditional.type = cudaGraphCondTypeWhile;
cParams.conditional.size = 1;
cParams.conditional.handle = handle;
cParams.conditional.type = cudaGraphCondTypeWhile;
cParams.conditional.size = 1;
checkCudaErrors(cudaGraphAddNode(&conditionalNode, graph, dependencies, numDependencies, &cParams));
cudaGraph_t bodyGraph = cParams.conditional.phGraph_out[0];
// Update stream capture dependencies to account for the node we manually added
checkCudaErrors(cudaStreamUpdateCaptureDependencies(captureStream, &conditionalNode, 1, cudaStreamSetCaptureDependencies));
checkCudaErrors(
cudaStreamUpdateCaptureDependencies(captureStream, &conditionalNode, 1, cudaStreamSetCaptureDependencies));
// Insert kernel node D
capturedWhileEmptyKernel<<<1, 1, 0, captureStream>>>();
@@ -306,7 +303,8 @@ void capturedWhileGraph(void)
cudaStream_t bodyStream;
checkCudaErrors(cudaStreamCreate(&bodyStream));
checkCudaErrors(cudaStreamBeginCaptureToGraph(bodyStream, bodyGraph, nullptr, nullptr, 0, cudaStreamCaptureModeGlobal));
checkCudaErrors(
cudaStreamBeginCaptureToGraph(bodyStream, bodyGraph, nullptr, nullptr, 0, cudaStreamCaptureModeGlobal));
// Insert kernel node C
capturedWhileKernel<<<1, 1, 0, bodyStream>>>(dPtr, handle);
@@ -351,15 +349,12 @@ void capturedWhileGraph(void)
*/
// This kernel will only be executed if the condition is false
__global__ void ifGraphKernelD(void)
{
printf("GPU: Hello from the GPU! The condition was false.\n");
}
__global__ void ifGraphKernelD(void) { printf("GPU: Hello from the GPU! The condition was false.\n"); }
// Setup and launch the graph
void simpleIfElseGraph(void)
{
cudaGraph_t graph;
cudaGraph_t graph;
cudaGraphExec_t graphExec;
cudaGraphNode_t kernelNode;
cudaGraphNode_t conditionalNode;
@@ -379,25 +374,25 @@ void simpleIfElseGraph(void)
// Use a kernel upstream of the conditional to set the handle value
cudaGraphNodeParams params = {cudaGraphNodeTypeKernel};
params.kernel.func = (void *)ifGraphKernelA;
params.kernel.func = (void *)ifGraphKernelA;
params.kernel.blockDim.x = params.kernel.blockDim.y = params.kernel.blockDim.z = 1;
params.kernel.gridDim.x = params.kernel.gridDim.y = params.kernel.gridDim.z = 1;
params.kernel.kernelParams = kernelArgs;
kernelArgs[0] = &dPtr;
kernelArgs[1] = &handle;
params.kernel.kernelParams = kernelArgs;
kernelArgs[0] = &dPtr;
kernelArgs[1] = &handle;
checkCudaErrors(cudaGraphAddNode(&kernelNode, graph, NULL, 0, &params));
cudaGraphNodeParams cParams = {cudaGraphNodeTypeConditional};
cParams.conditional.handle = handle;
cParams.conditional.type = cudaGraphCondTypeIf;
cParams.conditional.size = 2; // Set size to 2 to indicate an ELSE graph will be used
cParams.conditional.handle = handle;
cParams.conditional.type = cudaGraphCondTypeIf;
cParams.conditional.size = 2; // Set size to 2 to indicate an ELSE graph will be used
checkCudaErrors(cudaGraphAddNode(&conditionalNode, graph, &kernelNode, 1, &cParams));
cudaGraph_t bodyGraph = cParams.conditional.phGraph_out[0];
// Populate the body of the first graph in the conditional node, executed if the condition is true
cudaGraphNode_t trueBodyNode;
params.kernel.func = (void *)ifGraphKernelC;
params.kernel.func = (void *)ifGraphKernelC;
params.kernel.kernelParams = nullptr;
checkCudaErrors(cudaGraphAddNode(&trueBodyNode, bodyGraph, NULL, 0, &params));
@@ -405,7 +400,7 @@ void simpleIfElseGraph(void)
bodyGraph = cParams.conditional.phGraph_out[1];
cudaGraphNode_t falseBodyNode;
params.kernel.func = (void *)ifGraphKernelD;
params.kernel.func = (void *)ifGraphKernelD;
params.kernel.kernelParams = nullptr;
checkCudaErrors(cudaGraphAddNode(&falseBodyNode, bodyGraph, NULL, 0, &params));
@@ -452,30 +447,18 @@ __global__ void switchGraphKernelA(char *dPtr, cudaGraphConditionalHandle handle
printf("GPU: Handle set to %d\n", value);
}
__global__ void switchGraphKernelC(void)
{
printf("GPU: Hello from switchGraphKernelC(), running on the GPU!\n");
}
__global__ void switchGraphKernelC(void) { printf("GPU: Hello from switchGraphKernelC(), running on the GPU!\n"); }
__global__ void switchGraphKernelD(void)
{
printf("GPU: Hello from switchGraphKernelD(), running on the GPU!\n");
}
__global__ void switchGraphKernelD(void) { printf("GPU: Hello from switchGraphKernelD(), running on the GPU!\n"); }
__global__ void switchGraphKernelE(void)
{
printf("GPU: Hello from switchGraphKernelE(), running on the GPU!\n");
}
__global__ void switchGraphKernelE(void) { printf("GPU: Hello from switchGraphKernelE(), running on the GPU!\n"); }
__global__ void switchGraphKernelF(void)
{
printf("GPU: Hello from switchGraphKernelF(), running on the GPU!\n");
}
__global__ void switchGraphKernelF(void) { printf("GPU: Hello from switchGraphKernelF(), running on the GPU!\n"); }
// Setup and launch the graph
void simpleSwitchGraph(void)
{
cudaGraph_t graph;
cudaGraph_t graph;
cudaGraphExec_t graphExec;
cudaGraphNode_t kernelNode;
cudaGraphNode_t conditionalNode;
@@ -495,24 +478,24 @@ void simpleSwitchGraph(void)
// Use a kernel upstream of the conditional to set the handle value
cudaGraphNodeParams params = {cudaGraphNodeTypeKernel};
params.kernel.func = (void *)switchGraphKernelA;
params.kernel.func = (void *)switchGraphKernelA;
params.kernel.blockDim.x = params.kernel.blockDim.y = params.kernel.blockDim.z = 1;
params.kernel.gridDim.x = params.kernel.gridDim.y = params.kernel.gridDim.z = 1;
params.kernel.kernelParams = kernelArgs;
kernelArgs[0] = &dPtr;
kernelArgs[1] = &handle;
params.kernel.kernelParams = kernelArgs;
kernelArgs[0] = &dPtr;
kernelArgs[1] = &handle;
checkCudaErrors(cudaGraphAddNode(&kernelNode, graph, NULL, 0, &params));
cudaGraphNodeParams cParams = {cudaGraphNodeTypeConditional};
cParams.conditional.handle = handle;
cParams.conditional.type = cudaGraphCondTypeSwitch;
cParams.conditional.size = 4;
cParams.conditional.handle = handle;
cParams.conditional.type = cudaGraphCondTypeSwitch;
cParams.conditional.size = 4;
checkCudaErrors(cudaGraphAddNode(&conditionalNode, graph, &kernelNode, 1, &cParams));
// Populate the four graph bodies within the SWITCH conditional graph
cudaGraphNode_t bodyNode;
params.kernel.kernelParams = nullptr;
params.kernel.func = (void *)switchGraphKernelC;
params.kernel.func = (void *)switchGraphKernelC;
checkCudaErrors(cudaGraphAddNode(&bodyNode, cParams.conditional.phGraph_out[0], NULL, 0, &params));
params.kernel.func = (void *)switchGraphKernelD;
checkCudaErrors(cudaGraphAddNode(&bodyNode, cParams.conditional.phGraph_out[1], NULL, 0, &params));
@@ -523,8 +506,7 @@ void simpleSwitchGraph(void)
checkCudaErrors(cudaGraphInstantiate(&graphExec, graph, NULL, NULL, 0));
for (char i = 0; i < 5; i++)
{
for (char i = 0; i < 5; i++) {
// Initialize device memory and launch the graph
checkCudaErrors(cudaMemset(dPtr, i, 1));
printf("Host: Launching graph with device memory set to %d\n", i);
@@ -547,11 +529,9 @@ int main(int argc, char **argv)
int driverVersion = 0;
cudaDriverGetVersion(&driverVersion);
printf("Driver version is: %d.%d\n", driverVersion / 1000,
(driverVersion % 100) / 10);
printf("Driver version is: %d.%d\n", driverVersion / 1000, (driverVersion % 100) / 10);
if (driverVersion < 12030)
{
if (driverVersion < 12030) {
printf("Skipping execution as driver does not support Graph Conditional Nodes\n");
return 0;
}
@@ -560,8 +540,7 @@ int main(int argc, char **argv)
simpleDoWhileGraph();
capturedWhileGraph();
if (driverVersion < 12080)
{
if (driverVersion < 12080) {
printf("Skipping execution as driver does not support if/else and switch type Graph Conditional Nodes\n");
return 0;
}

603
Samples/3_CUDA_Features/graphMemoryFootprint/graphMemoryFootprint.cu
View File

@@ -33,375 +33,358 @@
#include <helper_cuda.h>
#include <helper_functions.h>
#define NUM_GRAPHS 8
#define NUM_GRAPHS 8
#define THREADS_PER_BLOCK 512
void printMemoryFootprint(int device) {
size_t footprint;
checkCudaErrors(cudaDeviceGetGraphMemAttribute(
device, (cudaGraphMemAttributeType)0, &footprint));
printf(" FOOTPRINT: %lu bytes\n", footprint);
void printMemoryFootprint(int device)
{
size_t footprint;
checkCudaErrors(cudaDeviceGetGraphMemAttribute(device, (cudaGraphMemAttributeType)0, &footprint));
printf(" FOOTPRINT: %lu bytes\n", footprint);
}
void prepareAllocParams(cudaMemAllocNodeParams *allocParams, size_t bytes,
int device) {
memset(allocParams, 0, sizeof(*allocParams));
void prepareAllocParams(cudaMemAllocNodeParams *allocParams, size_t bytes, int device)
{
memset(allocParams, 0, sizeof(*allocParams));
allocParams->bytesize = bytes;
allocParams->poolProps.allocType = cudaMemAllocationTypePinned;
allocParams->poolProps.location.id = device;
allocParams->poolProps.location.type = cudaMemLocationTypeDevice;
allocParams->bytesize = bytes;
allocParams->poolProps.allocType = cudaMemAllocationTypePinned;
allocParams->poolProps.location.id = device;
allocParams->poolProps.location.type = cudaMemLocationTypeDevice;
}
void createVirtAddrReuseGraph(cudaGraphExec_t *graphExec, size_t bytes,
int device) {
cudaGraph_t graph;
cudaGraphNode_t allocNodeA, allocNodeB, freeNodeA, freeNodeB;
cudaMemAllocNodeParams allocParams;
float *d_a, *d_b;
void createVirtAddrReuseGraph(cudaGraphExec_t *graphExec, size_t bytes, int device)
{
cudaGraph_t graph;
cudaGraphNode_t allocNodeA, allocNodeB, freeNodeA, freeNodeB;
cudaMemAllocNodeParams allocParams;
float *d_a, *d_b;
checkCudaErrors(cudaGraphCreate(&graph, 0));
prepareAllocParams(&allocParams, bytes, device);
checkCudaErrors(cudaGraphCreate(&graph, 0));
prepareAllocParams(&allocParams, bytes, device);
checkCudaErrors(
cudaGraphAddMemAllocNode(&allocNodeA, graph, NULL, 0, &allocParams));
d_a = (float *)allocParams.dptr;
checkCudaErrors(
cudaGraphAddMemFreeNode(&freeNodeA, graph, &allocNodeA, 1, (void *)d_a));
checkCudaErrors(cudaGraphAddMemAllocNode(&allocNodeA, graph, NULL, 0, &allocParams));
d_a = (float *)allocParams.dptr;
checkCudaErrors(cudaGraphAddMemFreeNode(&freeNodeA, graph, &allocNodeA, 1, (void *)d_a));
// The dependency between the allocation of d_b and the free of d_a allows d_b
// to reuse the same VA.
checkCudaErrors(cudaGraphAddMemAllocNode(&allocNodeB, graph, &freeNodeA, 1,
&allocParams));
d_b = (float *)allocParams.dptr;
// The dependency between the allocation of d_b and the free of d_a allows d_b
// to reuse the same VA.
checkCudaErrors(cudaGraphAddMemAllocNode(&allocNodeB, graph, &freeNodeA, 1, &allocParams));
d_b = (float *)allocParams.dptr;
if (d_a == d_b) {
printf("Check confirms that d_a and d_b share a virtual address.\n");
} else {
printf("Check shows that d_a and d_b DO NOT share a virtual address.\n");
}
if (d_a == d_b) {
printf("Check confirms that d_a and d_b share a virtual address.\n");
}
else {
printf("Check shows that d_a and d_b DO NOT share a virtual address.\n");
}
checkCudaErrors(
cudaGraphAddMemFreeNode(&freeNodeB, graph, &allocNodeB, 1, (void *)d_b));
checkCudaErrors(cudaGraphAddMemFreeNode(&freeNodeB, graph, &allocNodeB, 1, (void *)d_b));
checkCudaErrors(cudaGraphInstantiate(graphExec, graph, NULL, NULL, 0));
checkCudaErrors(cudaGraphDestroy(graph));
checkCudaErrors(cudaGraphInstantiate(graphExec, graph, NULL, NULL, 0));
checkCudaErrors(cudaGraphDestroy(graph));
}
void virtualAddressReuseSingleGraph(size_t bytes, int device) {
cudaStream_t stream;
cudaGraphExec_t graphExec;
void virtualAddressReuseSingleGraph(size_t bytes, int device)
{
cudaStream_t stream;
cudaGraphExec_t graphExec;
printf("================================\n");
printf("Running virtual address reuse example.\n");
printf(
"Sequential allocations & frees within a single graph enable CUDA to "
"reuse virtual addresses.\n\n");
printf("================================\n");
printf("Running virtual address reuse example.\n");
printf("Sequential allocations & frees within a single graph enable CUDA to "
"reuse virtual addresses.\n\n");
createVirtAddrReuseGraph(&graphExec, bytes, device);
checkCudaErrors(cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking));
createVirtAddrReuseGraph(&graphExec, bytes, device);
checkCudaErrors(cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking));
checkCudaErrors(cudaGraphLaunch(graphExec, stream));
checkCudaErrors(cudaStreamSynchronize(stream));
printMemoryFootprint(device);
checkCudaErrors(cudaGraphLaunch(graphExec, stream));
checkCudaErrors(cudaStreamSynchronize(stream));
printMemoryFootprint(device);
checkCudaErrors(cudaGraphExecDestroy(graphExec));
checkCudaErrors(cudaStreamDestroy(stream));
checkCudaErrors(cudaGraphExecDestroy(graphExec));
checkCudaErrors(cudaStreamDestroy(stream));
}
// This is a kernel that does no real work but runs at least for a specified
// number of clocks
__global__ void clockBlock(clock_t clock_count) {
unsigned int start_clock = (unsigned int)clock();
__global__ void clockBlock(clock_t clock_count)
{
unsigned int start_clock = (unsigned int)clock();
clock_t clock_offset = 0;
clock_t clock_offset = 0;
while (clock_offset < clock_count) {
unsigned int end_clock = (unsigned int)clock();
while (clock_offset < clock_count) {
unsigned int end_clock = (unsigned int)clock();
// The code below should work like
// this (thanks to modular arithmetics):
//
// clock_offset = (clock_t) (end_clock > start_clock ?
// end_clock - start_clock :
// end_clock + (0xffffffffu - start_clock));
//
// Indeed, let m = 2^32 then
// end - start = end + m - start (mod m).
// The code below should work like
// this (thanks to modular arithmetics):
//
// clock_offset = (clock_t) (end_clock > start_clock ?
// end_clock - start_clock :
// end_clock + (0xffffffffu - start_clock));
//
// Indeed, let m = 2^32 then
// end - start = end + m - start (mod m).
clock_offset = (clock_t)(end_clock - start_clock);
}
clock_offset = (clock_t)(end_clock - start_clock);
}
}
// A pointer to the allocated device buffer is returned in dPtr so the caller
// can compare virtual addresses. The kernel node is added to increase the
// graph's runtime.
void createSimpleAllocFreeGraph(cudaGraphExec_t *graphExec, float **dPtr,
size_t bytes, int device) {
cudaGraph_t graph;
cudaGraphNode_t allocNodeA, freeNodeA, blockDeviceNode;
cudaMemAllocNodeParams allocParams;
cudaKernelNodeParams blockDeviceNodeParams = {0};
int numElements = bytes / sizeof(float);
float kernelTime = 5; // time for each thread to run in microseconds
void createSimpleAllocFreeGraph(cudaGraphExec_t *graphExec, float **dPtr, size_t bytes, int device)
{
cudaGraph_t graph;
cudaGraphNode_t allocNodeA, freeNodeA, blockDeviceNode;
cudaMemAllocNodeParams allocParams;
cudaKernelNodeParams blockDeviceNodeParams = {0};
int numElements = bytes / sizeof(float);
float kernelTime = 5; // time for each thread to run in microseconds
checkCudaErrors(cudaGraphCreate(&graph, 0));
prepareAllocParams(&allocParams, bytes, device);
checkCudaErrors(cudaGraphCreate(&graph, 0));
prepareAllocParams(&allocParams, bytes, device);
checkCudaErrors(
cudaGraphAddMemAllocNode(&allocNodeA, graph, NULL, 0, &allocParams));
*dPtr = (float *)allocParams.dptr;
checkCudaErrors(cudaGraphAddMemAllocNode(&allocNodeA, graph, NULL, 0, &allocParams));
*dPtr = (float *)allocParams.dptr;
cudaDeviceProp deviceProp;
checkCudaErrors(cudaGetDeviceProperties(&deviceProp, device));
clock_t time_clocks = (clock_t)((kernelTime / 1000.0) * deviceProp.clockRate);
cudaDeviceProp deviceProp;
checkCudaErrors(cudaGetDeviceProperties(&deviceProp, device));
clock_t time_clocks = (clock_t)((kernelTime / 1000.0) * deviceProp.clockRate);
void *blockDeviceArgs[1] = {(void *)&time_clocks};
void *blockDeviceArgs[1] = {(void *)&time_clocks};
size_t numBlocks = numElements / (size_t)THREADS_PER_BLOCK;
blockDeviceNodeParams.gridDim = dim3(numBlocks, 1, 1);
blockDeviceNodeParams.blockDim = dim3(THREADS_PER_BLOCK, 1, 1);
blockDeviceNodeParams.sharedMemBytes = 0;
blockDeviceNodeParams.extra = NULL;
blockDeviceNodeParams.func = (void *)clockBlock;
blockDeviceNodeParams.kernelParams = (void **)blockDeviceArgs;
checkCudaErrors(cudaGraphAddKernelNode(&blockDeviceNode, graph, &allocNodeA,
1, &blockDeviceNodeParams));
size_t numBlocks = numElements / (size_t)THREADS_PER_BLOCK;
blockDeviceNodeParams.gridDim = dim3(numBlocks, 1, 1);
blockDeviceNodeParams.blockDim = dim3(THREADS_PER_BLOCK, 1, 1);
blockDeviceNodeParams.sharedMemBytes = 0;
blockDeviceNodeParams.extra = NULL;
blockDeviceNodeParams.func = (void *)clockBlock;
blockDeviceNodeParams.kernelParams = (void **)blockDeviceArgs;
checkCudaErrors(cudaGraphAddKernelNode(&blockDeviceNode, graph, &allocNodeA, 1, &blockDeviceNodeParams));
checkCudaErrors(cudaGraphAddMemFreeNode(&freeNodeA, graph, &blockDeviceNode,
1, (void *)*dPtr));
checkCudaErrors(cudaGraphAddMemFreeNode(&freeNodeA, graph, &blockDeviceNode, 1, (void *)*dPtr));
checkCudaErrors(cudaGraphInstantiate(graphExec, graph, NULL, NULL, 0));
checkCudaErrors(cudaGraphDestroy(graph));
checkCudaErrors(cudaGraphInstantiate(graphExec, graph, NULL, NULL, 0));
checkCudaErrors(cudaGraphDestroy(graph));
}
void physicalMemoryReuseSingleStream(size_t bytes, int device) {
cudaStream_t stream;
cudaGraphExec_t graphExecs[NUM_GRAPHS];
float *dPtrs[NUM_GRAPHS];
bool virtualAddrDiffer = true;
void physicalMemoryReuseSingleStream(size_t bytes, int device)
{
cudaStream_t stream;
cudaGraphExec_t graphExecs[NUM_GRAPHS];
float *dPtrs[NUM_GRAPHS];
bool virtualAddrDiffer = true;
printf("================================\n");
printf("Running physical memory reuse example.\n");
printf(
"CUDA reuses the same physical memory for allocations from separate "
"graphs when the allocation lifetimes don't overlap.\n\n");
printf("================================\n");
printf("Running physical memory reuse example.\n");
printf("CUDA reuses the same physical memory for allocations from separate "
"graphs when the allocation lifetimes don't overlap.\n\n");
for (int i = 0; i < NUM_GRAPHS; i++) {
createSimpleAllocFreeGraph(&graphExecs[i], &dPtrs[i], bytes, device);
}
printf("Creating the graph execs does not reserve any physical memory.\n");
printMemoryFootprint(device);
checkCudaErrors(cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking));
checkCudaErrors(cudaGraphLaunch(graphExecs[0], stream));
printf("\nThe first graph launched reserves the memory it needs.\n");
printMemoryFootprint(device);
checkCudaErrors(cudaGraphLaunch(graphExecs[0], stream));
printf(
"A subsequent launch of the same graph in the same stream reuses the "
"same physical memory. ");
printf("Thus the memory footprint does not grow here.\n");
printMemoryFootprint(device);
printf(
"\nSubsequent launches of other graphs in the same stream also reuse the "
"physical memory. ");
printf("Thus the memory footprint does not grow here.\n");
for (int i = 1; i < NUM_GRAPHS; i++) {
checkCudaErrors(cudaGraphLaunch(graphExecs[i], stream));
printf("%02d: ", i);
printMemoryFootprint(device);
}
checkCudaErrors(cudaStreamSynchronize(stream));
for (int i = 0; i < NUM_GRAPHS; i++) {
for (int j = i + 1; j < NUM_GRAPHS; j++) {
if (dPtrs[i] == dPtrs[j]) {
virtualAddrDiffer = false;
printf("Error: Graph exec %d and %d have the same virtual address!\n",
i - 1, i);
}
for (int i = 0; i < NUM_GRAPHS; i++) {
createSimpleAllocFreeGraph(&graphExecs[i], &dPtrs[i], bytes, device);
}
checkCudaErrors(cudaGraphExecDestroy(graphExecs[i]));
}
if (virtualAddrDiffer) {
printf("\nCheck confirms all graphs use a different virtual address.\n");
} else {
printf(
"\nAll graphs do NOT use different virtual addresses. Exiting test.\n");
exit(EXIT_FAILURE);
}
checkCudaErrors(cudaStreamDestroy(stream));
}
void simultaneousStreams(size_t bytes, int device) {
cudaStream_t streams[NUM_GRAPHS];
cudaGraphExec_t graphExecs[NUM_GRAPHS];
float *dPtrs[NUM_GRAPHS];
printf("================================\n");
printf("Running simultaneous streams example.\n");
printf("Graphs that can run concurrently need separate physical memory. ");
printf(
"In this example, each graph launched in a separate stream increases the "
"total memory footprint.\n\n");
printf(
"When launching a new graph, CUDA may reuse physical memory from a graph "
"whose execution has already ");
printf(
"finished -- even if the new graph is being launched in a different "
"stream from the completed graph. ");
printf(
"Therefore, a kernel node is added to the graphs to increase "
"runtime.\n\n");
for (int i = 0; i < NUM_GRAPHS; i++) {
createSimpleAllocFreeGraph(&graphExecs[i], &dPtrs[i], bytes, device);
checkCudaErrors(
cudaStreamCreateWithFlags(&streams[i], cudaStreamNonBlocking));
}
printf("Initial footprint:\n");
printMemoryFootprint(device);
printf(
"\nEach graph launch in a seperate stream grows the memory footprint:\n");
for (int i = 1; i < NUM_GRAPHS; i++) {
checkCudaErrors(cudaGraphLaunch(graphExecs[i], streams[i]));
printf("%02d: ", i);
printf("Creating the graph execs does not reserve any physical memory.\n");
printMemoryFootprint(device);
}
for (int i = 0; i < NUM_GRAPHS; i++) {
checkCudaErrors(cudaStreamSynchronize(streams[i]));
checkCudaErrors(cudaGraphExecDestroy(graphExecs[i]));
checkCudaErrors(cudaStreamDestroy(streams[i]));
}
}
checkCudaErrors(cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking));
void createSimpleAllocNoFreeGraph(cudaGraphExec_t *graphExec, float **dPtr,
size_t bytes, int device) {
cudaGraph_t graph;
cudaGraphNode_t allocNodeA;
cudaMemAllocNodeParams allocParams;
checkCudaErrors(cudaGraphCreate(&graph, 0));
prepareAllocParams(&allocParams, bytes, device);
checkCudaErrors(
cudaGraphAddMemAllocNode(&allocNodeA, graph, NULL, 0, &allocParams));
*dPtr = (float *)allocParams.dptr;
checkCudaErrors(cudaGraphInstantiate(graphExec, graph, NULL, NULL, 0));
checkCudaErrors(cudaGraphDestroy(graph));
}
void unfreedAllocations(size_t bytes, int device) {
cudaStream_t stream;
cudaGraphExec_t graphExecs[NUM_GRAPHS];
float *dPtrs[NUM_GRAPHS];
printf("================================\n");
printf("Running unfreed streams example.\n");
printf(
"CUDA cannot reuse phyiscal memory from graphs which do not free their "
"allocations.\n\n");
for (int i = 0; i < NUM_GRAPHS; i++) {
createSimpleAllocNoFreeGraph(&graphExecs[i], &dPtrs[i], bytes, device);
}
checkCudaErrors(cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking));
printf(
"Despite being launched in the same stream, each graph launch grows the "
"memory footprint. ");
printf(
"Since the allocation is not freed, CUDA keeps the memory valid for "
"use.\n");
for (int i = 0; i < NUM_GRAPHS; i++) {
checkCudaErrors(cudaGraphLaunch(graphExecs[i], stream));
printf("%02d: ", i);
checkCudaErrors(cudaGraphLaunch(graphExecs[0], stream));
printf("\nThe first graph launched reserves the memory it needs.\n");
printMemoryFootprint(device);
}
checkCudaErrors(cudaStreamSynchronize(stream));
checkCudaErrors(cudaGraphLaunch(graphExecs[0], stream));
printf("A subsequent launch of the same graph in the same stream reuses the "
"same physical memory. ");
printf("Thus the memory footprint does not grow here.\n");
printMemoryFootprint(device);
checkCudaErrors(cudaDeviceGraphMemTrim(device));
printf(
"\nTrimming does not impact the memory footprint since the un-freed "
"allocations are still holding onto the memory.\n");
printMemoryFootprint(device);
printf("\nSubsequent launches of other graphs in the same stream also reuse the "
"physical memory. ");
printf("Thus the memory footprint does not grow here.\n");
for (int i = 1; i < NUM_GRAPHS; i++) {
checkCudaErrors(cudaGraphLaunch(graphExecs[i], stream));
printf("%02d: ", i);
printMemoryFootprint(device);
}
for (int i = 0; i < NUM_GRAPHS; i++) {
checkCudaErrors(cudaFree(dPtrs[i]));
}
printf("\nFreeing the allocations does not shrink the footprint.\n");
printMemoryFootprint(device);
checkCudaErrors(cudaStreamSynchronize(stream));
checkCudaErrors(cudaDeviceGraphMemTrim(device));
printf(
"\nSince the allocations are now freed, trimming does reduce the "
"footprint even when the graph execs are not yet destroyed.\n");
printMemoryFootprint(device);
for (int i = 0; i < NUM_GRAPHS; i++) {
for (int j = i + 1; j < NUM_GRAPHS; j++) {
if (dPtrs[i] == dPtrs[j]) {
virtualAddrDiffer = false;
printf("Error: Graph exec %d and %d have the same virtual address!\n", i - 1, i);
}
}
checkCudaErrors(cudaGraphExecDestroy(graphExecs[i]));
}
if (virtualAddrDiffer) {
printf("\nCheck confirms all graphs use a different virtual address.\n");
}
else {
printf("\nAll graphs do NOT use different virtual addresses. Exiting test.\n");
exit(EXIT_FAILURE);
}
for (int i = 0; i < NUM_GRAPHS; i++) {
checkCudaErrors(cudaGraphExecDestroy(graphExecs[i]));
}
checkCudaErrors(cudaStreamDestroy(stream));
checkCudaErrors(cudaStreamDestroy(stream));
}
void cleanupMemory(int device) {
checkCudaErrors(cudaDeviceGraphMemTrim(device));
printf("\nCleaning up example by trimming device memory.\n");
printMemoryFootprint(device);
printf("\n");
void simultaneousStreams(size_t bytes, int device)
{
cudaStream_t streams[NUM_GRAPHS];
cudaGraphExec_t graphExecs[NUM_GRAPHS];
float *dPtrs[NUM_GRAPHS];
printf("================================\n");
printf("Running simultaneous streams example.\n");
printf("Graphs that can run concurrently need separate physical memory. ");
printf("In this example, each graph launched in a separate stream increases the "
"total memory footprint.\n\n");
printf("When launching a new graph, CUDA may reuse physical memory from a graph "
"whose execution has already ");
printf("finished -- even if the new graph is being launched in a different "
"stream from the completed graph. ");
printf("Therefore, a kernel node is added to the graphs to increase "
"runtime.\n\n");
for (int i = 0; i < NUM_GRAPHS; i++) {
createSimpleAllocFreeGraph(&graphExecs[i], &dPtrs[i], bytes, device);
checkCudaErrors(cudaStreamCreateWithFlags(&streams[i], cudaStreamNonBlocking));
}
printf("Initial footprint:\n");
printMemoryFootprint(device);
printf("\nEach graph launch in a seperate stream grows the memory footprint:\n");
for (int i = 1; i < NUM_GRAPHS; i++) {
checkCudaErrors(cudaGraphLaunch(graphExecs[i], streams[i]));
printf("%02d: ", i);
printMemoryFootprint(device);
}
for (int i = 0; i < NUM_GRAPHS; i++) {
checkCudaErrors(cudaStreamSynchronize(streams[i]));
checkCudaErrors(cudaGraphExecDestroy(graphExecs[i]));
checkCudaErrors(cudaStreamDestroy(streams[i]));
}
}
int main(int argc, char **argv) {
size_t bytes = 64 * 1024 * 1024;
int device = findCudaDevice(argc, (const char **)argv);
void createSimpleAllocNoFreeGraph(cudaGraphExec_t *graphExec, float **dPtr, size_t bytes, int device)
{
cudaGraph_t graph;
cudaGraphNode_t allocNodeA;
cudaMemAllocNodeParams allocParams;
int driverVersion = 0;
int deviceSupportsMemoryPools = 0;
checkCudaErrors(cudaGraphCreate(&graph, 0));
prepareAllocParams(&allocParams, bytes, device);
cudaDriverGetVersion(&driverVersion);
printf("Driver version is: %d.%d\n", driverVersion / 1000,
(driverVersion % 100) / 10);
checkCudaErrors(cudaGraphAddMemAllocNode(&allocNodeA, graph, NULL, 0, &allocParams));
*dPtr = (float *)allocParams.dptr;
if (driverVersion < 11040) {
printf("Waiving execution as driver does not support Graph Memory Nodes\n");
exit(EXIT_WAIVED);
}
cudaDeviceGetAttribute(&deviceSupportsMemoryPools,
cudaDevAttrMemoryPoolsSupported, device);
if (!deviceSupportsMemoryPools) {
printf("Waiving execution as device does not support Memory Pools\n");
exit(EXIT_WAIVED);
} else {
printf("Running sample.\n");
}
virtualAddressReuseSingleGraph(bytes, device);
cleanupMemory(device);
physicalMemoryReuseSingleStream(bytes, device);
cleanupMemory(device);
simultaneousStreams(bytes, device);
cleanupMemory(device);
unfreedAllocations(bytes, device);
cleanupMemory(device);
printf("================================\n");
printf("Sample complete.\n");
checkCudaErrors(cudaGraphInstantiate(graphExec, graph, NULL, NULL, 0));
checkCudaErrors(cudaGraphDestroy(graph));
}
void unfreedAllocations(size_t bytes, int device)
{
cudaStream_t stream;
cudaGraphExec_t graphExecs[NUM_GRAPHS];
float *dPtrs[NUM_GRAPHS];
printf("================================\n");
printf("Running unfreed streams example.\n");
printf("CUDA cannot reuse phyiscal memory from graphs which do not free their "
"allocations.\n\n");
for (int i = 0; i < NUM_GRAPHS; i++) {
createSimpleAllocNoFreeGraph(&graphExecs[i], &dPtrs[i], bytes, device);
}
checkCudaErrors(cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking));
printf("Despite being launched in the same stream, each graph launch grows the "
"memory footprint. ");
printf("Since the allocation is not freed, CUDA keeps the memory valid for "
"use.\n");
for (int i = 0; i < NUM_GRAPHS; i++) {
checkCudaErrors(cudaGraphLaunch(graphExecs[i], stream));
printf("%02d: ", i);
printMemoryFootprint(device);
}
checkCudaErrors(cudaStreamSynchronize(stream));
checkCudaErrors(cudaDeviceGraphMemTrim(device));
printf("\nTrimming does not impact the memory footprint since the un-freed "
"allocations are still holding onto the memory.\n");
printMemoryFootprint(device);
for (int i = 0; i < NUM_GRAPHS; i++) {
checkCudaErrors(cudaFree(dPtrs[i]));
}
printf("\nFreeing the allocations does not shrink the footprint.\n");
printMemoryFootprint(device);
checkCudaErrors(cudaDeviceGraphMemTrim(device));
printf("\nSince the allocations are now freed, trimming does reduce the "
"footprint even when the graph execs are not yet destroyed.\n");
printMemoryFootprint(device);
for (int i = 0; i < NUM_GRAPHS; i++) {
checkCudaErrors(cudaGraphExecDestroy(graphExecs[i]));
}
checkCudaErrors(cudaStreamDestroy(stream));
}
void cleanupMemory(int device)
{
checkCudaErrors(cudaDeviceGraphMemTrim(device));
printf("\nCleaning up example by trimming device memory.\n");
printMemoryFootprint(device);
printf("\n");
}
int main(int argc, char **argv)
{
size_t bytes = 64 * 1024 * 1024;
int device = findCudaDevice(argc, (const char **)argv);
int driverVersion = 0;
int deviceSupportsMemoryPools = 0;
cudaDriverGetVersion(&driverVersion);
printf("Driver version is: %d.%d\n", driverVersion / 1000, (driverVersion % 100) / 10);
if (driverVersion < 11040) {
printf("Waiving execution as driver does not support Graph Memory Nodes\n");
exit(EXIT_WAIVED);
}
cudaDeviceGetAttribute(&deviceSupportsMemoryPools, cudaDevAttrMemoryPoolsSupported, device);
if (!deviceSupportsMemoryPools) {
printf("Waiving execution as device does not support Memory Pools\n");
exit(EXIT_WAIVED);
}
else {
printf("Running sample.\n");
}
virtualAddressReuseSingleGraph(bytes, device);
cleanupMemory(device);
physicalMemoryReuseSingleStream(bytes, device);
cleanupMemory(device);
simultaneousStreams(bytes, device);
cleanupMemory(device);
unfreedAllocations(bytes, device);
cleanupMemory(device);
printf("================================\n");
printf("Sample complete.\n");
}

711
Samples/3_CUDA_Features/graphMemoryNodes/graphMemoryNodes.cu
View File

@@ -27,9 +27,8 @@
// System includes
#include <assert.h>
#include <stdio.h>
#include <climits>
#include <stdio.h>
#include <vector>
// CUDA runtime
@@ -39,81 +38,85 @@
#include <helper_cuda.h>
#include <helper_functions.h>
#define THREADS_PER_BLOCK 512
#define THREADS_PER_BLOCK 512
#define ALLOWABLE_VARIANCE 1.e-6f
#define NUM_ELEMENTS 8000000
#define NUM_ELEMENTS 8000000
// Stores the square of each input element in output array
__global__ void squareArray(const float *input, float *output,
int numElements) {
int idx = blockIdx.x * blockDim.x + threadIdx.x;
__global__ void squareArray(const float *input, float *output, int numElements)
{
int idx = blockIdx.x * blockDim.x + threadIdx.x;
if (idx < numElements) {
output[idx] = input[idx] * input[idx];
}
if (idx < numElements) {
output[idx] = input[idx] * input[idx];
}
}
// Stores the negative of each input element in output array
__global__ void negateArray(const float *input, float *output,
int numElements) {
int idx = blockIdx.x * blockDim.x + threadIdx.x;
__global__ void negateArray(const float *input, float *output, int numElements)
{
int idx = blockIdx.x * blockDim.x + threadIdx.x;
if (idx < numElements) {
output[idx] = input[idx] * -1;
}
if (idx < numElements) {
output[idx] = input[idx] * -1;
}
}
struct negSquareArrays {
float *input;
float *square;
float *negSquare;
int numElements;
size_t bytes;
size_t numBlocks;
struct negSquareArrays
{
float *input;
float *square;
float *negSquare;
int numElements;
size_t bytes;
size_t numBlocks;
};
void fillRandomly(float *array, int numElements) {
for (int n = 0; n < numElements; n++) {
array[n] = rand() / (float)RAND_MAX;
}
void fillRandomly(float *array, int numElements)
{
for (int n = 0; n < numElements; n++) {
array[n] = rand() / (float)RAND_MAX;
}
}
void resetOutputArrays(negSquareArrays *hostArrays) {
fillRandomly(hostArrays->square, hostArrays->numElements);
fillRandomly(hostArrays->negSquare, hostArrays->numElements);
void resetOutputArrays(negSquareArrays *hostArrays)
{
fillRandomly(hostArrays->square, hostArrays->numElements);
fillRandomly(hostArrays->negSquare, hostArrays->numElements);
}
void prepareHostArrays(negSquareArrays *hostArrays) {
hostArrays->numElements = NUM_ELEMENTS;
size_t bytes = hostArrays->numElements * sizeof(float);
void prepareHostArrays(negSquareArrays *hostArrays)
{
hostArrays->numElements = NUM_ELEMENTS;
size_t bytes = hostArrays->numElements * sizeof(float);
size_t numBlocks = hostArrays->numElements / (size_t)THREADS_PER_BLOCK;
if ((numBlocks % (size_t)THREADS_PER_BLOCK) != 0) {
numBlocks++;
}
size_t numBlocks = hostArrays->numElements / (size_t)THREADS_PER_BLOCK;
if ((numBlocks % (size_t)THREADS_PER_BLOCK) != 0) {
numBlocks++;
}
hostArrays->input = (float *)malloc(bytes);
hostArrays->square = (float *)malloc(bytes);
hostArrays->negSquare = (float *)malloc(bytes);
hostArrays->bytes = bytes;
hostArrays->numBlocks = numBlocks;
hostArrays->input = (float *)malloc(bytes);
hostArrays->square = (float *)malloc(bytes);
hostArrays->negSquare = (float *)malloc(bytes);
hostArrays->bytes = bytes;
hostArrays->numBlocks = numBlocks;
fillRandomly(hostArrays->input, hostArrays->numElements);
fillRandomly(hostArrays->square, hostArrays->numElements);
fillRandomly(hostArrays->negSquare, hostArrays->numElements);
fillRandomly(hostArrays->input, hostArrays->numElements);
fillRandomly(hostArrays->square, hostArrays->numElements);
fillRandomly(hostArrays->negSquare, hostArrays->numElements);
}
void createFreeGraph(cudaGraphExec_t *graphExec, float *dPtr) {
cudaGraph_t graph;
cudaGraphNode_t freeNode;
void createFreeGraph(cudaGraphExec_t *graphExec, float *dPtr)
{
cudaGraph_t graph;
cudaGraphNode_t freeNode;
checkCudaErrors(cudaGraphCreate(&graph, 0));
checkCudaErrors(cudaGraphCreate(&graph, 0));
checkCudaErrors(
cudaGraphAddMemFreeNode(&freeNode, graph, NULL, 0, (void *)dPtr));
checkCudaErrors(cudaGraphAddMemFreeNode(&freeNode, graph, NULL, 0, (void *)dPtr));
checkCudaErrors(cudaGraphInstantiate(graphExec, graph, NULL, NULL, 0));
checkCudaErrors(cudaGraphDestroy(graph));
checkCudaErrors(cudaGraphInstantiate(graphExec, graph, NULL, NULL, 0));
checkCudaErrors(cudaGraphDestroy(graph));
}
/**
@@ -145,115 +148,122 @@ void createFreeGraph(cudaGraphExec_t *graphExec, float *dPtr) {
* |
* free d_negSquare
*/
void createNegateSquaresGraphExplicitly(cudaGraphExec_t *graphExec, int device,
void createNegateSquaresGraphExplicitly(cudaGraphExec_t *graphExec,
int device,
negSquareArrays *hostArrays,
float **d_negSquare_out = NULL) {
// Array buffers on device
float *d_input, *d_square, *d_negSquare;
float **d_negSquare_out = NULL)
{
// Array buffers on device
float *d_input, *d_square, *d_negSquare;
// Memory allocation parameters
cudaMemAllocNodeParams allocParams;
memset(&allocParams, 0, sizeof(allocParams));
allocParams.bytesize = hostArrays->bytes;
allocParams.poolProps.allocType = cudaMemAllocationTypePinned;
allocParams.poolProps.location.id = device;
allocParams.poolProps.location.type = cudaMemLocationTypeDevice;
// Memory allocation parameters
cudaMemAllocNodeParams allocParams;
memset(&allocParams, 0, sizeof(allocParams));
allocParams.bytesize = hostArrays->bytes;
allocParams.poolProps.allocType = cudaMemAllocationTypePinned;
allocParams.poolProps.location.id = device;
allocParams.poolProps.location.type = cudaMemLocationTypeDevice;
// Kernel launch parameters
cudaKernelNodeParams kernelNodeParams = {0};
kernelNodeParams.gridDim = dim3(hostArrays->numBlocks, 1, 1);
kernelNodeParams.blockDim = dim3(THREADS_PER_BLOCK, 1, 1);
kernelNodeParams.sharedMemBytes = 0;
kernelNodeParams.extra = NULL;
// Kernel launch parameters
cudaKernelNodeParams kernelNodeParams = {0};
kernelNodeParams.gridDim = dim3(hostArrays->numBlocks, 1, 1);
kernelNodeParams.blockDim = dim3(THREADS_PER_BLOCK, 1, 1);
kernelNodeParams.sharedMemBytes = 0;
kernelNodeParams.extra = NULL;
cudaGraph_t graph;
cudaGraphNode_t allocNodeInput, allocNodeSquare, allocNodeNegSquare;
cudaGraphNode_t copyNodeInput, copyNodeSquare, copyNodeNegSquare;
cudaGraphNode_t squareKernelNode, negateKernelNode;
cudaGraphNode_t freeNodeInput, freeNodeSquare;
cudaGraph_t graph;
cudaGraphNode_t allocNodeInput, allocNodeSquare, allocNodeNegSquare;
cudaGraphNode_t copyNodeInput, copyNodeSquare, copyNodeNegSquare;
cudaGraphNode_t squareKernelNode, negateKernelNode;
cudaGraphNode_t freeNodeInput, freeNodeSquare;
// Buffer for storing graph node dependencies
std::vector<cudaGraphNode_t> nodeDependencies;
// Buffer for storing graph node dependencies
std::vector<cudaGraphNode_t> nodeDependencies;
checkCudaErrors(cudaGraphCreate(&graph, 0));
checkCudaErrors(cudaGraphCreate(&graph, 0));
checkCudaErrors(
cudaGraphAddMemAllocNode(&allocNodeInput, graph, NULL, 0, &allocParams));
d_input = (float *)allocParams.dptr;
checkCudaErrors(cudaGraphAddMemAllocNode(&allocNodeInput, graph, NULL, 0, &allocParams));
d_input = (float *)allocParams.dptr;
// To keep the graph structure simple (fewer branching dependencies),
// allocNodeSquare should depend on allocNodeInput
checkCudaErrors(cudaGraphAddMemAllocNode(&allocNodeSquare, graph,
&allocNodeInput, 1, &allocParams));
d_square = (float *)allocParams.dptr;
// To keep the graph structure simple (fewer branching dependencies),
// allocNodeSquare should depend on allocNodeInput
checkCudaErrors(cudaGraphAddMemAllocNode(&allocNodeSquare, graph, &allocNodeInput, 1, &allocParams));
d_square = (float *)allocParams.dptr;
// copyNodeInput needs to depend on allocNodeInput because copyNodeInput
// writes to d_input. It does so here indirectly through allocNodeSquare.
checkCudaErrors(cudaGraphAddMemcpyNode1D(
&copyNodeInput, graph, &allocNodeSquare, 1, d_input, hostArrays->input,
hostArrays->bytes, cudaMemcpyHostToDevice));
// copyNodeInput needs to depend on allocNodeInput because copyNodeInput
// writes to d_input. It does so here indirectly through allocNodeSquare.
checkCudaErrors(cudaGraphAddMemcpyNode1D(&copyNodeInput,
graph,
&allocNodeSquare,
1,
d_input,
hostArrays->input,
hostArrays->bytes,
cudaMemcpyHostToDevice));
void *squareKernelArgs[3] = {(void *)&d_input, (void *)&d_square,
(void *)&(hostArrays->numElements)};
kernelNodeParams.func = (void *)squareArray;
kernelNodeParams.kernelParams = (void **)squareKernelArgs;
void *squareKernelArgs[3] = {(void *)&d_input, (void *)&d_square, (void *)&(hostArrays->numElements)};
kernelNodeParams.func = (void *)squareArray;
kernelNodeParams.kernelParams = (void **)squareKernelArgs;
// Square kernel depends on copyNodeInput to ensure all data is on the device
// before kernel launch.
checkCudaErrors(cudaGraphAddKernelNode(&squareKernelNode, graph,
&copyNodeInput, 1, &kernelNodeParams));
// Square kernel depends on copyNodeInput to ensure all data is on the device
// before kernel launch.
checkCudaErrors(cudaGraphAddKernelNode(&squareKernelNode, graph, &copyNodeInput, 1, &kernelNodeParams));
checkCudaErrors(cudaGraphAddMemcpyNode1D(
&copyNodeSquare, graph, &squareKernelNode, 1, hostArrays->square,
d_square, hostArrays->bytes, cudaMemcpyDeviceToHost));
checkCudaErrors(cudaGraphAddMemcpyNode1D(&copyNodeSquare,
graph,
&squareKernelNode,
1,
hostArrays->square,
d_square,
hostArrays->bytes,
cudaMemcpyDeviceToHost));
// Free of d_input depends on the square kernel to ensure that d_input is not
// freed while being read by the kernel. It also depends on the alloc of
// d_input via squareKernelNode > copyNodeInput > allocNodeSquare >
// allocNodeInput.
checkCudaErrors(cudaGraphAddMemFreeNode(&freeNodeInput, graph,
&squareKernelNode, 1, d_input));
// Free of d_input depends on the square kernel to ensure that d_input is not
// freed while being read by the kernel. It also depends on the alloc of
// d_input via squareKernelNode > copyNodeInput > allocNodeSquare >
// allocNodeInput.
checkCudaErrors(cudaGraphAddMemFreeNode(&freeNodeInput, graph, &squareKernelNode, 1, d_input));
// Allocation of C depends on free of A so CUDA can reuse the virtual address.
checkCudaErrors(cudaGraphAddMemAllocNode(&allocNodeNegSquare, graph,
&freeNodeInput, 1, &allocParams));
d_negSquare = (float *)allocParams.dptr;
// Allocation of C depends on free of A so CUDA can reuse the virtual address.
checkCudaErrors(cudaGraphAddMemAllocNode(&allocNodeNegSquare, graph, &freeNodeInput, 1, &allocParams));
d_negSquare = (float *)allocParams.dptr;
if (d_negSquare == d_input) {
printf(
"Check verified that d_negSquare and d_input share a virtual "
"address.\n");
}
if (d_negSquare == d_input) {
printf("Check verified that d_negSquare and d_input share a virtual "
"address.\n");
}
void *negateKernelArgs[3] = {(void *)&d_square, (void *)&d_negSquare,
(void *)&(hostArrays->numElements)};
kernelNodeParams.func = (void *)negateArray;
kernelNodeParams.kernelParams = (void **)negateKernelArgs;
void *negateKernelArgs[3] = {(void *)&d_square, (void *)&d_negSquare, (void *)&(hostArrays->numElements)};
kernelNodeParams.func = (void *)negateArray;
kernelNodeParams.kernelParams = (void **)negateKernelArgs;
checkCudaErrors(cudaGraphAddKernelNode(
&negateKernelNode, graph, &allocNodeNegSquare, 1, &kernelNodeParams));
checkCudaErrors(cudaGraphAddKernelNode(&negateKernelNode, graph, &allocNodeNegSquare, 1, &kernelNodeParams));
nodeDependencies.push_back(copyNodeSquare);
nodeDependencies.push_back(negateKernelNode);
checkCudaErrors(cudaGraphAddMemFreeNode(&freeNodeSquare, graph,
nodeDependencies.data(),
nodeDependencies.size(), d_square));
nodeDependencies.clear();
nodeDependencies.push_back(copyNodeSquare);
nodeDependencies.push_back(negateKernelNode);
checkCudaErrors(
cudaGraphAddMemFreeNode(&freeNodeSquare, graph, nodeDependencies.data(), nodeDependencies.size(), d_square));
nodeDependencies.clear();
checkCudaErrors(cudaGraphAddMemcpyNode1D(
&copyNodeNegSquare, graph, &negateKernelNode, 1, hostArrays->negSquare,
d_negSquare, hostArrays->bytes, cudaMemcpyDeviceToHost));
checkCudaErrors(cudaGraphAddMemcpyNode1D(&copyNodeNegSquare,
graph,
&negateKernelNode,
1,
hostArrays->negSquare,
d_negSquare,
hostArrays->bytes,
cudaMemcpyDeviceToHost));
if (d_negSquare_out == NULL) {
cudaGraphNode_t freeNodeNegSquare;
checkCudaErrors(cudaGraphAddMemFreeNode(
&freeNodeNegSquare, graph, &copyNodeNegSquare, 1, d_negSquare));
} else {
*d_negSquare_out = d_negSquare;
}
if (d_negSquare_out == NULL) {
cudaGraphNode_t freeNodeNegSquare;
checkCudaErrors(cudaGraphAddMemFreeNode(&freeNodeNegSquare, graph, &copyNodeNegSquare, 1, d_negSquare));
}
else {
*d_negSquare_out = d_negSquare;
}
checkCudaErrors(cudaGraphInstantiate(graphExec, graph, NULL, NULL, 0));
checkCudaErrors(cudaGraphDestroy(graph));
checkCudaErrors(cudaGraphInstantiate(graphExec, graph, NULL, NULL, 0));
checkCudaErrors(cudaGraphDestroy(graph));
}
/**
@@ -293,60 +303,55 @@ void createNegateSquaresGraphExplicitly(cudaGraphExec_t *graphExec, int device,
* | |
* wait squareFreeEvent --------------<---- record squareFreeEvent
*/
void doNegateSquaresInStream(cudaStream_t stream1, negSquareArrays *hostArrays,
float **d_negSquare_out = NULL) {
float *d_input, *d_square, *d_negSquare;
cudaStream_t stream2;
cudaEvent_t squareKernelCompleteEvent, negateKernelCompleteEvent,
squareFreeEvent;
void doNegateSquaresInStream(cudaStream_t stream1, negSquareArrays *hostArrays, float **d_negSquare_out = NULL)
{
float *d_input, *d_square, *d_negSquare;
cudaStream_t stream2;
cudaEvent_t squareKernelCompleteEvent, negateKernelCompleteEvent, squareFreeEvent;
checkCudaErrors(cudaStreamCreateWithFlags(&stream2, cudaStreamNonBlocking));
checkCudaErrors(cudaStreamCreateWithFlags(&stream2, cudaStreamNonBlocking));
checkCudaErrors(cudaEventCreate(&squareKernelCompleteEvent));
checkCudaErrors(cudaEventCreate(&negateKernelCompleteEvent));
checkCudaErrors(cudaEventCreate(&squareFreeEvent));
checkCudaErrors(cudaEventCreate(&squareKernelCompleteEvent));
checkCudaErrors(cudaEventCreate(&negateKernelCompleteEvent));
checkCudaErrors(cudaEventCreate(&squareFreeEvent));
// Virtual addresses are assigned synchronously when cudaMallocAsync is
// called, thus there is no performace benefit gained by separating the
// allocations into two streams.
checkCudaErrors(cudaMallocAsync(&d_input, hostArrays->bytes, stream1));
checkCudaErrors(cudaMallocAsync(&d_square, hostArrays->bytes, stream1));
// Virtual addresses are assigned synchronously when cudaMallocAsync is
// called, thus there is no performace benefit gained by separating the
// allocations into two streams.
checkCudaErrors(cudaMallocAsync(&d_input, hostArrays->bytes, stream1));
checkCudaErrors(cudaMallocAsync(&d_square, hostArrays->bytes, stream1));
checkCudaErrors(cudaMemcpyAsync(d_input, hostArrays->input, hostArrays->bytes,
cudaMemcpyHostToDevice, stream1));
squareArray<<<hostArrays->numBlocks, THREADS_PER_BLOCK, 0, stream1>>>(
d_input, d_square, hostArrays->numElements);
checkCudaErrors(cudaEventRecord(squareKernelCompleteEvent, stream1));
checkCudaErrors(cudaMemcpyAsync(d_input, hostArrays->input, hostArrays->bytes, cudaMemcpyHostToDevice, stream1));
squareArray<<<hostArrays->numBlocks, THREADS_PER_BLOCK, 0, stream1>>>(d_input, d_square, hostArrays->numElements);
checkCudaErrors(cudaEventRecord(squareKernelCompleteEvent, stream1));
checkCudaErrors(cudaStreamWaitEvent(stream2, squareKernelCompleteEvent, 0));
checkCudaErrors(cudaMemcpyAsync(hostArrays->square, d_square,
hostArrays->bytes, cudaMemcpyDeviceToHost,
stream2));
checkCudaErrors(cudaStreamWaitEvent(stream2, squareKernelCompleteEvent, 0));
checkCudaErrors(cudaMemcpyAsync(hostArrays->square, d_square, hostArrays->bytes, cudaMemcpyDeviceToHost, stream2));
checkCudaErrors(cudaFreeAsync(d_input, stream1));
checkCudaErrors(cudaMallocAsync(&d_negSquare, hostArrays->bytes, stream1));
negateArray<<<hostArrays->numBlocks, THREADS_PER_BLOCK, 0, stream1>>>(
d_square, d_negSquare, hostArrays->numElements);
checkCudaErrors(cudaEventRecord(negateKernelCompleteEvent, stream1));
checkCudaErrors(cudaMemcpyAsync(hostArrays->negSquare, d_negSquare,
hostArrays->bytes, cudaMemcpyDeviceToHost,
stream1));
if (d_negSquare_out == NULL) {
checkCudaErrors(cudaFreeAsync(d_negSquare, stream1));
} else {
*d_negSquare_out = d_negSquare;
}
checkCudaErrors(cudaFreeAsync(d_input, stream1));
checkCudaErrors(cudaMallocAsync(&d_negSquare, hostArrays->bytes, stream1));
negateArray<<<hostArrays->numBlocks, THREADS_PER_BLOCK, 0, stream1>>>(
d_square, d_negSquare, hostArrays->numElements);
checkCudaErrors(cudaEventRecord(negateKernelCompleteEvent, stream1));
checkCudaErrors(
cudaMemcpyAsync(hostArrays->negSquare, d_negSquare, hostArrays->bytes, cudaMemcpyDeviceToHost, stream1));
if (d_negSquare_out == NULL) {
checkCudaErrors(cudaFreeAsync(d_negSquare, stream1));
}
else {
*d_negSquare_out = d_negSquare;
}
checkCudaErrors(cudaStreamWaitEvent(stream2, negateKernelCompleteEvent, 0));
checkCudaErrors(cudaFreeAsync(d_square, stream2));
checkCudaErrors(cudaEventRecord(squareFreeEvent, stream2));
checkCudaErrors(cudaStreamWaitEvent(stream2, negateKernelCompleteEvent, 0));
checkCudaErrors(cudaFreeAsync(d_square, stream2));
checkCudaErrors(cudaEventRecord(squareFreeEvent, stream2));
checkCudaErrors(cudaStreamWaitEvent(stream1, squareFreeEvent, 0));
checkCudaErrors(cudaStreamWaitEvent(stream1, squareFreeEvent, 0));
checkCudaErrors(cudaStreamDestroy(stream2));
checkCudaErrors(cudaEventDestroy(squareKernelCompleteEvent));
checkCudaErrors(cudaEventDestroy(negateKernelCompleteEvent));
checkCudaErrors(cudaEventDestroy(squareFreeEvent));
checkCudaErrors(cudaStreamDestroy(stream2));
checkCudaErrors(cudaEventDestroy(squareKernelCompleteEvent));
checkCudaErrors(cudaEventDestroy(negateKernelCompleteEvent));
checkCudaErrors(cudaEventDestroy(squareFreeEvent));
}
/**
@@ -356,199 +361,195 @@ void doNegateSquaresInStream(cudaStream_t stream1, negSquareArrays *hostArrays,
*/
void createNegateSquaresGraphWithStreamCapture(cudaGraphExec_t *graphExec,
negSquareArrays *hostArrays,
float **d_negSquare_out = NULL) {
cudaGraph_t graph;
cudaStream_t stream;
float **d_negSquare_out = NULL)
{
cudaGraph_t graph;
cudaStream_t stream;
checkCudaErrors(cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking));
checkCudaErrors(cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking));
checkCudaErrors(cudaStreamBeginCapture(stream, cudaStreamCaptureModeGlobal));
doNegateSquaresInStream(stream, hostArrays, d_negSquare_out);
checkCudaErrors(cudaStreamEndCapture(stream, &graph));
checkCudaErrors(cudaStreamBeginCapture(stream, cudaStreamCaptureModeGlobal));
doNegateSquaresInStream(stream, hostArrays, d_negSquare_out);
checkCudaErrors(cudaStreamEndCapture(stream, &graph));
checkCudaErrors(cudaGraphInstantiate(graphExec, graph, NULL, NULL, 0));
checkCudaErrors(cudaStreamDestroy(stream));
checkCudaErrors(cudaGraphDestroy(graph));
checkCudaErrors(cudaGraphInstantiate(graphExec, graph, NULL, NULL, 0));
checkCudaErrors(cudaStreamDestroy(stream));
checkCudaErrors(cudaGraphDestroy(graph));
}
void prepareRefArrays(negSquareArrays *hostArrays,
negSquareArrays *deviceRefArrays,
bool **foundValidationFailure) {
deviceRefArrays->bytes = hostArrays->bytes;
deviceRefArrays->numElements = hostArrays->numElements;
void prepareRefArrays(negSquareArrays *hostArrays, negSquareArrays *deviceRefArrays, bool **foundValidationFailure)
{
deviceRefArrays->bytes = hostArrays->bytes;
deviceRefArrays->numElements = hostArrays->numElements;
for (int i = 0; i < hostArrays->numElements; i++) {
hostArrays->square[i] = hostArrays->input[i] * hostArrays->input[i];
hostArrays->negSquare[i] = hostArrays->square[i] * -1;
}
checkCudaErrors(
cudaMalloc((void **)&deviceRefArrays->negSquare, deviceRefArrays->bytes));
checkCudaErrors(cudaMemcpy(deviceRefArrays->negSquare, hostArrays->negSquare,
hostArrays->bytes, cudaMemcpyHostToDevice));
checkCudaErrors(
cudaMallocManaged((void **)foundValidationFailure, sizeof(bool)));
}
int checkValidationFailure(bool *foundValidationFailure) {
if (*foundValidationFailure) {
printf("Validation FAILURE!\n\n");
*foundValidationFailure = false;
return EXIT_FAILURE;
} else {
printf("Validation PASSED!\n\n");
return EXIT_SUCCESS;
}
}
__global__ void validateGPU(float *d_negSquare, negSquareArrays devRefArrays,
bool *foundValidationFailure) {
int idx = blockIdx.x * blockDim.x + threadIdx.x;
float ref, diff;
if (idx < devRefArrays.numElements) {
ref = devRefArrays.negSquare[idx];
diff = d_negSquare[idx] - ref;
diff *= diff;
ref *= ref;
if (diff / ref > ALLOWABLE_VARIANCE) {
*foundValidationFailure = true;
for (int i = 0; i < hostArrays->numElements; i++) {
hostArrays->square[i] = hostArrays->input[i] * hostArrays->input[i];
hostArrays->negSquare[i] = hostArrays->square[i] * -1;
}
}
checkCudaErrors(cudaMalloc((void **)&deviceRefArrays->negSquare, deviceRefArrays->bytes));
checkCudaErrors(
cudaMemcpy(deviceRefArrays->negSquare, hostArrays->negSquare, hostArrays->bytes, cudaMemcpyHostToDevice));
checkCudaErrors(cudaMallocManaged((void **)foundValidationFailure, sizeof(bool)));
}
void validateHost(negSquareArrays *hostArrays, bool *foundValidationFailure) {
float ref, diff;
for (int i = 0; i < hostArrays->numElements; i++) {
ref = hostArrays->input[i] * hostArrays->input[i] * -1;
diff = hostArrays->negSquare[i] - ref;
diff *= diff;
ref *= ref;
if (diff / ref > ALLOWABLE_VARIANCE) {
*foundValidationFailure = true;
int checkValidationFailure(bool *foundValidationFailure)
{
if (*foundValidationFailure) {
printf("Validation FAILURE!\n\n");
*foundValidationFailure = false;
return EXIT_FAILURE;
}
else {
printf("Validation PASSED!\n\n");
return EXIT_SUCCESS;
}
}
}
int main(int argc, char **argv) {
negSquareArrays hostArrays, deviceRefArrays;
cudaStream_t stream;
cudaGraphExec_t graphExec, graphExecFreeC;
__global__ void validateGPU(float *d_negSquare, negSquareArrays devRefArrays, bool *foundValidationFailure)
{
int idx = blockIdx.x * blockDim.x + threadIdx.x;
float ref, diff;
// Declare pointers for GPU buffers
float *d_negSquare = NULL;
bool *foundValidationFailure = NULL;
if (idx < devRefArrays.numElements) {
ref = devRefArrays.negSquare[idx];
diff = d_negSquare[idx] - ref;
diff *= diff;
ref *= ref;
if (diff / ref > ALLOWABLE_VARIANCE) {
*foundValidationFailure = true;
}
}
}
srand(time(0));
int device = findCudaDevice(argc, (const char **)argv);
void validateHost(negSquareArrays *hostArrays, bool *foundValidationFailure)
{
float ref, diff;
int driverVersion = 0;
int deviceSupportsMemoryPools = 0;
for (int i = 0; i < hostArrays->numElements; i++) {
ref = hostArrays->input[i] * hostArrays->input[i] * -1;
diff = hostArrays->negSquare[i] - ref;
diff *= diff;
ref *= ref;
if (diff / ref > ALLOWABLE_VARIANCE) {
*foundValidationFailure = true;
}
}
}
cudaDriverGetVersion(&driverVersion);
printf("Driver version is: %d.%d\n", driverVersion / 1000,
(driverVersion % 100) / 10);
int main(int argc, char **argv)
{
negSquareArrays hostArrays, deviceRefArrays;
cudaStream_t stream;
cudaGraphExec_t graphExec, graphExecFreeC;
if (driverVersion < 11040) {
printf("Waiving execution as driver does not support Graph Memory Nodes\n");
exit(EXIT_WAIVED);
}
// Declare pointers for GPU buffers
float *d_negSquare = NULL;
bool *foundValidationFailure = NULL;
cudaDeviceGetAttribute(&deviceSupportsMemoryPools,
cudaDevAttrMemoryPoolsSupported, device);
if (!deviceSupportsMemoryPools) {
printf("Waiving execution as device does not support Memory Pools\n");
exit(EXIT_WAIVED);
} else {
printf("Setting up sample.\n");
}
srand(time(0));
int device = findCudaDevice(argc, (const char **)argv);
prepareHostArrays(&hostArrays);
prepareRefArrays(&hostArrays, &deviceRefArrays, &foundValidationFailure);
checkCudaErrors(cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking));
printf("Setup complete.\n\n");
int driverVersion = 0;
int deviceSupportsMemoryPools = 0;
printf("Running negateSquares in a stream.\n");
doNegateSquaresInStream(stream, &hostArrays);
checkCudaErrors(cudaStreamSynchronize(stream));
printf("Validating negateSquares in a stream...\n");
validateHost(&hostArrays, foundValidationFailure);
checkValidationFailure(foundValidationFailure);
resetOutputArrays(&hostArrays);
cudaDriverGetVersion(&driverVersion);
printf("Driver version is: %d.%d\n", driverVersion / 1000, (driverVersion % 100) / 10);
printf("Running negateSquares in a stream-captured graph.\n");
createNegateSquaresGraphWithStreamCapture(&graphExec, &hostArrays);
checkCudaErrors(cudaGraphLaunch(graphExec, stream));
checkCudaErrors(cudaStreamSynchronize(stream));
printf("Validating negateSquares in a stream-captured graph...\n");
validateHost(&hostArrays, foundValidationFailure);
checkValidationFailure(foundValidationFailure);
resetOutputArrays(&hostArrays);
if (driverVersion < 11040) {
printf("Waiving execution as driver does not support Graph Memory Nodes\n");
exit(EXIT_WAIVED);
}
printf("Running negateSquares in an explicitly constructed graph.\n");
createNegateSquaresGraphExplicitly(&graphExec, device, &hostArrays);
checkCudaErrors(cudaGraphLaunch(graphExec, stream));
checkCudaErrors(cudaStreamSynchronize(stream));
printf("Validating negateSquares in an explicitly constructed graph...\n");
validateHost(&hostArrays, foundValidationFailure);
checkValidationFailure(foundValidationFailure);
resetOutputArrays(&hostArrays);
cudaDeviceGetAttribute(&deviceSupportsMemoryPools, cudaDevAttrMemoryPoolsSupported, device);
if (!deviceSupportsMemoryPools) {
printf("Waiving execution as device does not support Memory Pools\n");
exit(EXIT_WAIVED);
}
else {
printf("Setting up sample.\n");
}
// Each of the three examples below free d_negSquare outside the graph. As
// demonstrated by validateGPU, d_negSquare can be accessed by outside the
// graph before d_negSquare is freed.
prepareHostArrays(&hostArrays);
prepareRefArrays(&hostArrays, &deviceRefArrays, &foundValidationFailure);
checkCudaErrors(cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking));
printf("Setup complete.\n\n");
printf("Running negateSquares with d_negSquare freed outside the stream.\n");
createNegateSquaresGraphExplicitly(&graphExec, device, &hostArrays,
&d_negSquare);
checkCudaErrors(cudaGraphLaunch(graphExec, stream));
validateGPU<<<hostArrays.numBlocks, THREADS_PER_BLOCK, 0, stream>>>(
d_negSquare, deviceRefArrays, foundValidationFailure);
// Since cudaFree is synchronous, the stream must synchronize before freeing
// d_negSquare to ensure d_negSquare no longer being accessed.
checkCudaErrors(cudaStreamSynchronize(stream));
checkCudaErrors(cudaFree(d_negSquare));
printf(
"Validating negateSquares with d_negSquare freed outside the "
"stream...\n");
validateHost(&hostArrays, foundValidationFailure);
checkValidationFailure(foundValidationFailure);
resetOutputArrays(&hostArrays);
printf("Running negateSquares in a stream.\n");
doNegateSquaresInStream(stream, &hostArrays);
checkCudaErrors(cudaStreamSynchronize(stream));
printf("Validating negateSquares in a stream...\n");
validateHost(&hostArrays, foundValidationFailure);
checkValidationFailure(foundValidationFailure);
resetOutputArrays(&hostArrays);
printf("Running negateSquares with d_negSquare freed outside the graph.\n");
checkCudaErrors(cudaGraphLaunch(graphExec, stream));
validateGPU<<<hostArrays.numBlocks, THREADS_PER_BLOCK, 0, stream>>>(
d_negSquare, deviceRefArrays, foundValidationFailure);
checkCudaErrors(cudaFreeAsync(d_negSquare, stream));
checkCudaErrors(cudaStreamSynchronize(stream));
printf(
"Validating negateSquares with d_negSquare freed outside the graph...\n");
checkValidationFailure(foundValidationFailure);
resetOutputArrays(&hostArrays);
printf("Running negateSquares in a stream-captured graph.\n");
createNegateSquaresGraphWithStreamCapture(&graphExec, &hostArrays);
checkCudaErrors(cudaGraphLaunch(graphExec, stream));
checkCudaErrors(cudaStreamSynchronize(stream));
printf("Validating negateSquares in a stream-captured graph...\n");
validateHost(&hostArrays, foundValidationFailure);
checkValidationFailure(foundValidationFailure);
resetOutputArrays(&hostArrays);
printf(
"Running negateSquares with d_negSquare freed in a different graph.\n");
createFreeGraph(&graphExecFreeC, d_negSquare);
checkCudaErrors(cudaGraphLaunch(graphExec, stream));
validateGPU<<<hostArrays.numBlocks, THREADS_PER_BLOCK, 0, stream>>>(
d_negSquare, deviceRefArrays, foundValidationFailure);
checkCudaErrors(cudaGraphLaunch(graphExecFreeC, stream));
checkCudaErrors(cudaStreamSynchronize(stream));
printf(
"Validating negateSquares with d_negSquare freed in a different "
"graph...\n");
checkValidationFailure(foundValidationFailure);
printf("Running negateSquares in an explicitly constructed graph.\n");
createNegateSquaresGraphExplicitly(&graphExec, device, &hostArrays);
checkCudaErrors(cudaGraphLaunch(graphExec, stream));
checkCudaErrors(cudaStreamSynchronize(stream));
printf("Validating negateSquares in an explicitly constructed graph...\n");
validateHost(&hostArrays, foundValidationFailure);
checkValidationFailure(foundValidationFailure);
resetOutputArrays(&hostArrays);
printf("Cleaning up sample.\n");
checkCudaErrors(cudaGraphExecDestroy(graphExec));
checkCudaErrors(cudaGraphExecDestroy(graphExecFreeC));
checkCudaErrors(cudaStreamDestroy(stream));
checkCudaErrors(cudaFree(foundValidationFailure));
checkCudaErrors(cudaFree(deviceRefArrays.negSquare));
free(hostArrays.input);
free(hostArrays.square);
free(hostArrays.negSquare);
printf("Cleanup complete. Exiting sample.\n");
}
// Each of the three examples below free d_negSquare outside the graph. As
// demonstrated by validateGPU, d_negSquare can be accessed by outside the
// graph before d_negSquare is freed.
printf("Running negateSquares with d_negSquare freed outside the stream.\n");
createNegateSquaresGraphExplicitly(&graphExec, device, &hostArrays, &d_negSquare);
checkCudaErrors(cudaGraphLaunch(graphExec, stream));
validateGPU<<<hostArrays.numBlocks, THREADS_PER_BLOCK, 0, stream>>>(
d_negSquare, deviceRefArrays, foundValidationFailure);
// Since cudaFree is synchronous, the stream must synchronize before freeing
// d_negSquare to ensure d_negSquare no longer being accessed.
checkCudaErrors(cudaStreamSynchronize(stream));
checkCudaErrors(cudaFree(d_negSquare));
printf("Validating negateSquares with d_negSquare freed outside the "
"stream...\n");
validateHost(&hostArrays, foundValidationFailure);
checkValidationFailure(foundValidationFailure);
resetOutputArrays(&hostArrays);
printf("Running negateSquares with d_negSquare freed outside the graph.\n");
checkCudaErrors(cudaGraphLaunch(graphExec, stream));
validateGPU<<<hostArrays.numBlocks, THREADS_PER_BLOCK, 0, stream>>>(
d_negSquare, deviceRefArrays, foundValidationFailure);
checkCudaErrors(cudaFreeAsync(d_negSquare, stream));
checkCudaErrors(cudaStreamSynchronize(stream));
printf("Validating negateSquares with d_negSquare freed outside the graph...\n");
checkValidationFailure(foundValidationFailure);
resetOutputArrays(&hostArrays);
printf("Running negateSquares with d_negSquare freed in a different graph.\n");
createFreeGraph(&graphExecFreeC, d_negSquare);
checkCudaErrors(cudaGraphLaunch(graphExec, stream));
validateGPU<<<hostArrays.numBlocks, THREADS_PER_BLOCK, 0, stream>>>(
d_negSquare, deviceRefArrays, foundValidationFailure);
checkCudaErrors(cudaGraphLaunch(graphExecFreeC, stream));
checkCudaErrors(cudaStreamSynchronize(stream));
printf("Validating negateSquares with d_negSquare freed in a different "
"graph...\n");
checkValidationFailure(foundValidationFailure);
printf("Cleaning up sample.\n");
checkCudaErrors(cudaGraphExecDestroy(graphExec));
checkCudaErrors(cudaGraphExecDestroy(graphExecFreeC));
checkCudaErrors(cudaStreamDestroy(stream));
checkCudaErrors(cudaFree(foundValidationFailure));
checkCudaErrors(cudaFree(deviceRefArrays.negSquare));
free(hostArrays.input);
free(hostArrays.square);
free(hostArrays.negSquare);
printf("Cleanup complete. Exiting sample.\n");
}

813
Samples/3_CUDA_Features/immaTensorCoreGemm/immaTensorCoreGemm.cu
View File

@@ -115,7 +115,7 @@
// Implementation constants.
#define WARPS_PER_BLOCK 8
#define WARPS_PER_BLOCK 8
#define THREADS_PER_BLOCK (WARP_SIZE * WARPS_PER_BLOCK)
#if SHARED_MEMORY_LIMIT_64K
@@ -132,10 +132,10 @@
#define CHUNK_K 16
#endif
#define CHUNK_LINE_BYTES (CHUNK_K * K * sizeof(uint8_t))
#define WARP_COPY_BYTES (WARP_SIZE * sizeof(int4))
#define CHUNK_LINE_BYTES (CHUNK_K * K * sizeof(uint8_t))
#define WARP_COPY_BYTES (WARP_SIZE * sizeof(int4))
#define CHUNK_COPY_LINES_PER_WARP (WARP_COPY_BYTES / CHUNK_LINE_BYTES)
#define CHUNK_COPY_LINE_LANES (WARP_SIZE / CHUNK_COPY_LINES_PER_WARP)
#define CHUNK_COPY_LINE_LANES (WARP_SIZE / CHUNK_COPY_LINES_PER_WARP)
#define BLOCK_ROW_WARPS 2
#define BLOCK_COL_WARPS 4
@@ -166,236 +166,219 @@
// nvcuda::wmma::load_matrix_sync.
#define SKEW_UINT8 32
#define checkKernelErrors(expr) \
do { \
expr; \
\
cudaError_t __err = cudaGetLastError(); \
if (__err != cudaSuccess) { \
printf("Line %d: '%s' failed: %s\n", __LINE__, #expr, \
cudaGetErrorString(__err)); \
abort(); \
} \
} while (0)
#define checkKernelErrors(expr) \
do { \
expr; \
\
cudaError_t __err = cudaGetLastError(); \
if (__err != cudaSuccess) { \
printf("Line %d: '%s' failed: %s\n", __LINE__, #expr, cudaGetErrorString(__err)); \
abort(); \
} \
} while (0)
using namespace nvcuda;
__host__ void init_host_matrices(uint8_t *a, uint8_t *b, int *c) {
for (int i = 0; i < M_GLOBAL; i++) {
for (int j = 0; j < K_GLOBAL; j++) {
a[i * K_GLOBAL + j] = (uint8_t)(rand() % 3);
__host__ void init_host_matrices(uint8_t *a, uint8_t *b, int *c)
{
for (int i = 0; i < M_GLOBAL; i++) {
for (int j = 0; j < K_GLOBAL; j++) {
a[i * K_GLOBAL + j] = (uint8_t)(rand() % 3);
}
}
}
for (int i = 0; i < N_GLOBAL; i++) {
for (int j = 0; j < K_GLOBAL; j++) {
b[i * K_GLOBAL + j] = (uint8_t)(rand() % 3);
for (int i = 0; i < N_GLOBAL; i++) {
for (int j = 0; j < K_GLOBAL; j++) {
b[i * K_GLOBAL + j] = (uint8_t)(rand() % 3);
}
}
}
for (int t = 0; t < M_GLOBAL * N_GLOBAL; t++) {
c[t] = (rand() % 3);
}
for (int t = 0; t < M_GLOBAL * N_GLOBAL; t++) {
c[t] = (rand() % 3);
}
}
__global__ void compute_gemm_imma(const uint8_t *A, const uint8_t *B,
const int *C, int *D, int alpha, int beta) {
extern __shared__ uint8_t shmem[][CHUNK_K * K + SKEW_UINT8];
__global__ void compute_gemm_imma(const uint8_t *A, const uint8_t *B, const int *C, int *D, int alpha, int beta)
{
extern __shared__ uint8_t shmem[][CHUNK_K * K + SKEW_UINT8];
// Warp and lane identification.
const unsigned int warpId = threadIdx.x / WARP_SIZE;
const unsigned int laneId = threadIdx.x % WARP_SIZE;
// Warp and lane identification.
const unsigned int warpId = threadIdx.x / WARP_SIZE;
const unsigned int laneId = threadIdx.x % WARP_SIZE;
// Offset in shared memory from which the B matrix is stored.
const size_t shmem_idx_b_off = BLOCK_COL_TILES * M;
// Offset in shared memory from which the B matrix is stored.
const size_t shmem_idx_b_off = BLOCK_COL_TILES * M;
// This pointer is used to access the C and D matrix tiles this warp computes.
int *shmem_warp_tile_ptr = (int *)&shmem[0][0] +
(warpId / 2) * SHMEM_STRIDE * K * 2 +
(warpId % 2) * SHMEM_OFFSET;
// This pointer is used to access the C and D matrix tiles this warp computes.
int *shmem_warp_tile_ptr = (int *)&shmem[0][0] + (warpId / 2) * SHMEM_STRIDE * K * 2 + (warpId % 2) * SHMEM_OFFSET;
// This pointer is used to stream the C and D matrices block-wide tile to and
// from shared memory.
int *shmem_warp_stream_ptr = (int *)&shmem[0][0] + warpId * SHMEM_STRIDE * K;
// This pointer is used to stream the C and D matrices block-wide tile to and
// from shared memory.
int *shmem_warp_stream_ptr = (int *)&shmem[0][0] + warpId * SHMEM_STRIDE * K;
// Adjust the beta scaler, as it'll be multiplied by alpha at the end of
// each tile computation. Technically this is not generally correct (may
// result in a loss of precision). Zero still needs to be specially handled
// though.
beta /= alpha;
// Adjust the beta scaler, as it'll be multiplied by alpha at the end of
// each tile computation. Technically this is not generally correct (may
// result in a loss of precision). Zero still needs to be specially handled
// though.
beta /= alpha;
// Each CTA slides along the 128 x 128 tiles from the top left corner of the
// matrix to the right and down, and selects the next tile to compute. Once
// there's no such tile, all warps in this CTA exit.
for (unsigned int block_pos = blockIdx.x;; block_pos += gridDim.x) {
const unsigned int block_tile_i =
((block_pos * BLOCK_ROW_TILES) / N_TILES) * (BLOCK_COL_TILES);
const unsigned int block_tile_j = (block_pos * BLOCK_COL_TILES) % N_TILES;
// Each CTA slides along the 128 x 128 tiles from the top left corner of the
// matrix to the right and down, and selects the next tile to compute. Once
// there's no such tile, all warps in this CTA exit.
for (unsigned int block_pos = blockIdx.x;; block_pos += gridDim.x) {
const unsigned int block_tile_i = ((block_pos * BLOCK_ROW_TILES) / N_TILES) * (BLOCK_COL_TILES);
const unsigned int block_tile_j = (block_pos * BLOCK_COL_TILES) % N_TILES;
// Stop when there are no more D matrix tiles to compute in this CTA.
if (block_tile_i >= M_TILES) {
break;
}
// This warp's pointer to the C matrix data to copy memory from to shared
// memory.
const size_t gmem_idx =
(block_tile_i + warpId) * M * GLOBAL_MEM_STRIDE + block_tile_j * N;
const int *src_gmem_warp_stream_ptr = &C[gmem_idx];
// Stream multiple C tiles to shared memory.
#pragma unroll
for (int i = 0; i < K; i++) {
typedef int4 copy_t;
*((copy_t *)(shmem_warp_stream_ptr + SHMEM_STRIDE * i) + laneId) =
*((copy_t *)(src_gmem_warp_stream_ptr + GLOBAL_MEM_STRIDE * i) +
laneId);
}
__syncthreads();
// These fragments will accumulate the result of A and B matrix fragment
// multiplications along the K_GLOBAL dimension.
wmma::fragment<wmma::accumulator, M, N, K, int> c[WARP_COL_TILES]
[WARP_ROW_TILES];
// Load the C matrix tiles into fragments from shared memory.
#pragma unroll
for (int i = 0; i < WARP_COL_TILES; i++) {
#pragma unroll
for (int j = 0; j < WARP_ROW_TILES; j++) {
const int *tile_ptr =
shmem_warp_tile_ptr + i * SHMEM_STRIDE * K + j * N;
wmma::load_matrix_sync(c[i][j], tile_ptr, SHMEM_STRIDE, C_LAYOUT);
}
}
__syncthreads();
// Scale the C matrix.
#pragma unroll
for (int i = 0; i < WARP_COL_TILES; i++) {
#pragma unroll
for (int j = 0; j < WARP_ROW_TILES; j++) {
#pragma unroll
for (int t = 0; t < c[i][j].num_elements; t++) {
c[i][j].x[t] *= beta;
// Stop when there are no more D matrix tiles to compute in this CTA.
if (block_tile_i >= M_TILES) {
break;
}
}
}
// Select what warp copies what matrix to shared memory.
// Warps 0-3 copy the A matrix, warps 4-7 copy the B matrix.
const uint8_t *warp_ptr = (warpId < 4) ? (&A[block_tile_i * M * K_GLOBAL] +
M * K_GLOBAL * (warpId % 4) * 2)
: (&B[block_tile_j * N * K_GLOBAL] +
N * K_GLOBAL * (warpId % 4) * 2);
// This warp's pointer to the C matrix data to copy memory from to shared
// memory.
const size_t gmem_idx = (block_tile_i + warpId) * M * GLOBAL_MEM_STRIDE + block_tile_j * N;
const int *src_gmem_warp_stream_ptr = &C[gmem_idx];
// Go through the global K dimension by a fixed step at a time.
// Stream multiple C tiles to shared memory.
#pragma unroll
for (int tile_k = 0; tile_k < K_TILES; tile_k += CHUNK_K) {
// Copy slices of the A and B matrices to shared memory.
// The first half of the warps in the CTA copy the A matrix, the rest copy
// the B matrix.
size_t shmem_idx =
warpId < (WARPS_PER_BLOCK / 2)
? (M * (warpId % (WARPS_PER_BLOCK / 2)) * 2)
: (N * (warpId % (WARPS_PER_BLOCK / 2)) * 2 + shmem_idx_b_off);
for (int i = 0; i < K; i++) {
typedef int4 copy_t;
// First half of the warp copies the first row / column of the matrix,
// the second half of the warp copies the next.
int4 *lane_ptr = (int4 *)(warp_ptr + tile_k * K +
(laneId / CHUNK_COPY_LINE_LANES) * K_GLOBAL) +
(laneId % CHUNK_COPY_LINE_LANES);
*((copy_t *)(shmem_warp_stream_ptr + SHMEM_STRIDE * i) + laneId) =
*((copy_t *)(src_gmem_warp_stream_ptr + GLOBAL_MEM_STRIDE * i) + laneId);
}
// Shift the second half of the warp to the next row / column in the
// shared memory.
shmem_idx += laneId / CHUNK_COPY_LINE_LANES;
__syncthreads();
#pragma unroll
for (int i = 0; i < ((WARP_SIZE / 2) / CHUNK_COPY_LINES_PER_WARP) * 2;
i++) {
// Copy 16 bytes at once in each lane.
*((int4 *)&shmem[shmem_idx][0] + (laneId % CHUNK_COPY_LINE_LANES)) =
*lane_ptr;
// Advance the global memory pointer and the shared memory index.
lane_ptr = (int4 *)((uint8_t *)lane_ptr +
K_GLOBAL * CHUNK_COPY_LINES_PER_WARP);
shmem_idx += CHUNK_COPY_LINES_PER_WARP;
}
__syncthreads();
// Compute a grid of C matrix tiles in each warp.
#pragma unroll
for (int k_step = 0; k_step < CHUNK_K; k_step++) {
wmma::fragment<wmma::matrix_a, M, N, K, uint8_t, wmma::row_major>
a[WARP_COL_TILES];
wmma::fragment<wmma::matrix_b, M, N, K, uint8_t, wmma::col_major>
b[WARP_ROW_TILES];
// These fragments will accumulate the result of A and B matrix fragment
// multiplications along the K_GLOBAL dimension.
wmma::fragment<wmma::accumulator, M, N, K, int> c[WARP_COL_TILES][WARP_ROW_TILES];
// Load the C matrix tiles into fragments from shared memory.
#pragma unroll
for (int i = 0; i < WARP_COL_TILES; i++) {
size_t shmem_idx_a = (warpId / 2) * M * 2 + (i * M);
const uint8_t *tile_ptr = &shmem[shmem_idx_a][k_step * K];
#pragma unroll
for (int j = 0; j < WARP_ROW_TILES; j++) {
const int *tile_ptr = shmem_warp_tile_ptr + i * SHMEM_STRIDE * K + j * N;
wmma::load_matrix_sync(a[i], tile_ptr, K * CHUNK_K + SKEW_UINT8);
wmma::load_matrix_sync(c[i][j], tile_ptr, SHMEM_STRIDE, C_LAYOUT);
}
}
__syncthreads();
// Scale the C matrix.
#pragma unroll
for (int i = 0; i < WARP_COL_TILES; i++) {
#pragma unroll
for (int j = 0; j < WARP_ROW_TILES; j++) {
#pragma unroll
for (int t = 0; t < c[i][j].num_elements; t++) {
c[i][j].x[t] *= beta;
}
}
}
// Select what warp copies what matrix to shared memory.
// Warps 0-3 copy the A matrix, warps 4-7 copy the B matrix.
const uint8_t *warp_ptr = (warpId < 4) ? (&A[block_tile_i * M * K_GLOBAL] + M * K_GLOBAL * (warpId % 4) * 2)
: (&B[block_tile_j * N * K_GLOBAL] + N * K_GLOBAL * (warpId % 4) * 2);
// Go through the global K dimension by a fixed step at a time.
#pragma unroll
for (int tile_k = 0; tile_k < K_TILES; tile_k += CHUNK_K) {
// Copy slices of the A and B matrices to shared memory.
// The first half of the warps in the CTA copy the A matrix, the rest copy
// the B matrix.
size_t shmem_idx = warpId < (WARPS_PER_BLOCK / 2)
? (M * (warpId % (WARPS_PER_BLOCK / 2)) * 2)
: (N * (warpId % (WARPS_PER_BLOCK / 2)) * 2 + shmem_idx_b_off);
// First half of the warp copies the first row / column of the matrix,
// the second half of the warp copies the next.
int4 *lane_ptr = (int4 *)(warp_ptr + tile_k * K + (laneId / CHUNK_COPY_LINE_LANES) * K_GLOBAL)
+ (laneId % CHUNK_COPY_LINE_LANES);
// Shift the second half of the warp to the next row / column in the
// shared memory.
shmem_idx += laneId / CHUNK_COPY_LINE_LANES;
#pragma unroll
for (int j = 0; j < WARP_ROW_TILES; j++) {
if (i == 0) {
// Load the B matrix fragment once, because it is going to be
// reused against the other A matrix fragments.
size_t shmem_idx_b = shmem_idx_b_off +
(WARP_ROW_TILES * N) * (warpId % 2) +
(j * N);
const uint8_t *tile_ptr = &shmem[shmem_idx_b][k_step * K];
for (int i = 0; i < ((WARP_SIZE / 2) / CHUNK_COPY_LINES_PER_WARP) * 2; i++) {
// Copy 16 bytes at once in each lane.
*((int4 *)&shmem[shmem_idx][0] + (laneId % CHUNK_COPY_LINE_LANES)) = *lane_ptr;
wmma::load_matrix_sync(b[j], tile_ptr, K * CHUNK_K + SKEW_UINT8);
// Advance the global memory pointer and the shared memory index.
lane_ptr = (int4 *)((uint8_t *)lane_ptr + K_GLOBAL * CHUNK_COPY_LINES_PER_WARP);
shmem_idx += CHUNK_COPY_LINES_PER_WARP;
}
wmma::mma_sync(c[i][j], a[i], b[j], c[i][j]);
}
__syncthreads();
// Compute a grid of C matrix tiles in each warp.
#pragma unroll
for (int k_step = 0; k_step < CHUNK_K; k_step++) {
wmma::fragment<wmma::matrix_a, M, N, K, uint8_t, wmma::row_major> a[WARP_COL_TILES];
wmma::fragment<wmma::matrix_b, M, N, K, uint8_t, wmma::col_major> b[WARP_ROW_TILES];
#pragma unroll
for (int i = 0; i < WARP_COL_TILES; i++) {
size_t shmem_idx_a = (warpId / 2) * M * 2 + (i * M);
const uint8_t *tile_ptr = &shmem[shmem_idx_a][k_step * K];
wmma::load_matrix_sync(a[i], tile_ptr, K * CHUNK_K + SKEW_UINT8);
#pragma unroll
for (int j = 0; j < WARP_ROW_TILES; j++) {
if (i == 0) {
// Load the B matrix fragment once, because it is going to be
// reused against the other A matrix fragments.
size_t shmem_idx_b = shmem_idx_b_off + (WARP_ROW_TILES * N) * (warpId % 2) + (j * N);
const uint8_t *tile_ptr = &shmem[shmem_idx_b][k_step * K];
wmma::load_matrix_sync(b[j], tile_ptr, K * CHUNK_K + SKEW_UINT8);
}
wmma::mma_sync(c[i][j], a[i], b[j], c[i][j]);
}
}
}
__syncthreads();
}
}
__syncthreads();
// Store the D fragments to shared memory.
#pragma unroll
for (int i = 0; i < WARP_COL_TILES; i++) {
#pragma unroll
for (int j = 0; j < WARP_ROW_TILES; j++) {
#pragma unroll
// Uniform, point-wise transformations of ALL fragment elements by ALL
// threads in the warp are well-defined even though element indices
// within fragment storage are not defined.
for (int t = 0; t < c[i][j].num_elements; t++)
c[i][j].x[t] *= alpha;
int *tile_ptr = shmem_warp_tile_ptr + i * SHMEM_STRIDE * K + j * N;
wmma::store_matrix_sync(tile_ptr, c[i][j], SHMEM_STRIDE, C_LAYOUT);
}
}
__syncthreads();
// Now that shared memory contains all the D tiles, stream them to global
// memory.
int *dst_gmem_warp_stream_ptr = &D[gmem_idx];
#pragma unroll
for (int i = 0; i < K; i++) {
*((int4 *)(dst_gmem_warp_stream_ptr + GLOBAL_MEM_STRIDE * i) + laneId) =
*((int4 *)(shmem_warp_stream_ptr + SHMEM_STRIDE * i) + laneId);
}
__syncthreads();
}
// Store the D fragments to shared memory.
#pragma unroll
for (int i = 0; i < WARP_COL_TILES; i++) {
#pragma unroll
for (int j = 0; j < WARP_ROW_TILES; j++) {
#pragma unroll
// Uniform, point-wise transformations of ALL fragment elements by ALL
// threads in the warp are well-defined even though element indices
// within fragment storage are not defined.
for (int t = 0; t < c[i][j].num_elements; t++) c[i][j].x[t] *= alpha;
int *tile_ptr = shmem_warp_tile_ptr + i * SHMEM_STRIDE * K + j * N;
wmma::store_matrix_sync(tile_ptr, c[i][j], SHMEM_STRIDE, C_LAYOUT);
}
}
__syncthreads();
// Now that shared memory contains all the D tiles, stream them to global
// memory.
int *dst_gmem_warp_stream_ptr = &D[gmem_idx];
#pragma unroll
for (int i = 0; i < K; i++) {
*((int4 *)(dst_gmem_warp_stream_ptr + GLOBAL_MEM_STRIDE * i) + laneId) =
*((int4 *)(shmem_warp_stream_ptr + SHMEM_STRIDE * i) + laneId);
}
__syncthreads();
}
}
// Performs an MxNxK GEMM (C=alpha*A*B + beta*C) assuming:
@@ -406,250 +389,244 @@ __global__ void compute_gemm_imma(const uint8_t *A, const uint8_t *B,
// is designed for
// demonstration purposes only to show the CUDA WMMA API use without
// relying on availability of the shared memory.
__global__ void simple_wmma_gemm_imma(const uint8_t *a, const uint8_t *b,
const int *c, int *d, int m_ld, int n_ld,
int k_ld, int alpha, int beta) {
// Leading dimensions. Packed with no transpositions.
int lda = m_ld;
int ldb = k_ld;
int ldc = n_ld;
__global__ void simple_wmma_gemm_imma(const uint8_t *a,
const uint8_t *b,
const int *c,
int *d,
int m_ld,
int n_ld,
int k_ld,
int alpha,
int beta)
{
// Leading dimensions. Packed with no transpositions.
int lda = m_ld;
int ldb = k_ld;
int ldc = n_ld;
// Tile using a 2D grid
int warpM = (blockIdx.x * blockDim.x + threadIdx.x) / warpSize;
int warpN = (blockIdx.y * blockDim.y + threadIdx.y);
// Tile using a 2D grid
int warpM = (blockIdx.x * blockDim.x + threadIdx.x) / warpSize;
int warpN = (blockIdx.y * blockDim.y + threadIdx.y);
// Declare the fragments
wmma::fragment<wmma::matrix_a, WMMA_M, WMMA_N, WMMA_K, uint8_t,
wmma::row_major>
a_frag;
wmma::fragment<wmma::matrix_b, WMMA_M, WMMA_N, WMMA_K, uint8_t,
wmma::col_major>
b_frag;
wmma::fragment<wmma::accumulator, WMMA_M, WMMA_N, WMMA_K, int> acc_frag;
wmma::fragment<wmma::accumulator, WMMA_M, WMMA_N, WMMA_K, int> c_frag;
// Declare the fragments
wmma::fragment<wmma::matrix_a, WMMA_M, WMMA_N, WMMA_K, uint8_t, wmma::row_major> a_frag;
wmma::fragment<wmma::matrix_b, WMMA_M, WMMA_N, WMMA_K, uint8_t, wmma::col_major> b_frag;
wmma::fragment<wmma::accumulator, WMMA_M, WMMA_N, WMMA_K, int> acc_frag;
wmma::fragment<wmma::accumulator, WMMA_M, WMMA_N, WMMA_K, int> c_frag;
wmma::fill_fragment(acc_frag, 0.0f);
wmma::fill_fragment(acc_frag, 0.0f);
// Loop over k
for (int i = 0; i < k_ld; i += WMMA_K) {
int aCol = i;
int aRow = warpM * WMMA_M;
// Loop over k
for (int i = 0; i < k_ld; i += WMMA_K) {
int aCol = i;
int aRow = warpM * WMMA_M;
int bCol = i;
int bRow = warpN * WMMA_N;
int bCol = i;
int bRow = warpN * WMMA_N;
// Bounds checking
if (aRow < m_ld && aCol < k_ld && bRow < k_ld && bCol < n_ld) {
// Load the inputs
wmma::load_matrix_sync(a_frag, a + aCol + aRow * lda, lda);
wmma::load_matrix_sync(b_frag, b + bCol + bRow * ldb, ldb);
// Bounds checking
if (aRow < m_ld && aCol < k_ld && bRow < k_ld && bCol < n_ld) {
// Load the inputs
wmma::load_matrix_sync(a_frag, a + aCol + aRow * lda, lda);
wmma::load_matrix_sync(b_frag, b + bCol + bRow * ldb, ldb);
// Perform the matrix multiplication
wmma::mma_sync(acc_frag, a_frag, b_frag, acc_frag);
}
}
// Load in the current value of c, scale it by beta, and add this our result
// scaled by alpha
int cCol = warpN * WMMA_N;
int cRow = warpM * WMMA_M;
if (cRow < m_ld && cCol < n_ld) {
wmma::load_matrix_sync(c_frag, c + cCol + cRow * ldc, ldc,
wmma::mem_row_major);
for (int i = 0; i < c_frag.num_elements; i++) {
c_frag.x[i] = alpha * acc_frag.x[i] + beta * c_frag.x[i];
// Perform the matrix multiplication
wmma::mma_sync(acc_frag, a_frag, b_frag, acc_frag);
}
}
// Store the output
wmma::store_matrix_sync(d + cCol + cRow * ldc, c_frag, ldc,
wmma::mem_row_major);
}
// Load in the current value of c, scale it by beta, and add this our result
// scaled by alpha
int cCol = warpN * WMMA_N;
int cRow = warpM * WMMA_M;
if (cRow < m_ld && cCol < n_ld) {
wmma::load_matrix_sync(c_frag, c + cCol + cRow * ldc, ldc, wmma::mem_row_major);
for (int i = 0; i < c_frag.num_elements; i++) {
c_frag.x[i] = alpha * acc_frag.x[i] + beta * c_frag.x[i];
}
// Store the output
wmma::store_matrix_sync(d + cCol + cRow * ldc, c_frag, ldc, wmma::mem_row_major);
}
}
__host__ void matMultiplyOnHost(uint8_t *A, uint8_t *B, int *C, int alpha,
int beta, int numARows, int numAColumns,
int numBRows, int numBColumns, int numCRows,
int numCColumns) {
for (int i = 0; i < numCRows; i++) {
for (int j = 0; j < numCColumns; j++) {
int temp = 0;
__host__ void matMultiplyOnHost(uint8_t *A,
uint8_t *B,
int *C,
int alpha,
int beta,
int numARows,
int numAColumns,
int numBRows,
int numBColumns,
int numCRows,
int numCColumns)
{
for (int i = 0; i < numCRows; i++) {
for (int j = 0; j < numCColumns; j++) {
int temp = 0;
for (int k = 0; k < numAColumns; k++) {
temp += A[i * numAColumns + k] * B[j * numBRows + k];
}
for (int k = 0; k < numAColumns; k++) {
temp += A[i * numAColumns + k] * B[j * numBRows + k];
}
C[i * numCColumns + j] = temp * alpha + beta * C[i * numCColumns + j];
C[i * numCColumns + j] = temp * alpha + beta * C[i * numCColumns + j];
}
}
}
}
int main(int argc, char **argv) {
printf("Initializing...\n");
int main(int argc, char **argv)
{
printf("Initializing...\n");
int dev = findCudaDevice(argc, (const char **)argv);
int dev = findCudaDevice(argc, (const char **)argv);
cudaDeviceProp deviceProp;
checkCudaErrors(cudaGetDeviceProperties(&deviceProp, dev));
cudaDeviceProp deviceProp;
checkCudaErrors(cudaGetDeviceProperties(&deviceProp, dev));
// Tensor cores require a GPU of Volta (SM72) architecture or higher.
if (deviceProp.major < 7 || (deviceProp.major <= 7 && deviceProp.minor < 2)) {
printf(
"immaTensorCoreGemm requires SM 7.2 or higher to use Tensor Cores. "
"Exiting...\n");
exit(EXIT_WAIVED);
}
printf("M: %d (%d x %d)\n", M_GLOBAL, M, M_TILES);
printf("N: %d (%d x %d)\n", N_GLOBAL, N, N_TILES);
printf("K: %d (%d x %d)\n", K_GLOBAL, K, K_TILES);
uint8_t *A_h = NULL;
uint8_t *B_h = NULL;
int *C_h = NULL;
#if CPU_DEBUG
int *result_hD = NULL;
int *result_host = NULL;
#endif
A_h = (uint8_t *)malloc(sizeof(uint8_t) * M_GLOBAL * K_GLOBAL);
B_h = (uint8_t *)malloc(sizeof(uint8_t) * K_GLOBAL * N_GLOBAL);
C_h = (int *)malloc(sizeof(int) * M_GLOBAL * N_GLOBAL);
#if CPU_DEBUG
result_hD = (int *)malloc(sizeof(int) * M_GLOBAL * N_GLOBAL);
result_host = (int *)malloc(sizeof(int) * M_GLOBAL * N_GLOBAL);
#endif
uint8_t *A = NULL;
uint8_t *B = NULL;
int *C = NULL;
int *D = NULL;
checkCudaErrors(
cudaMalloc(reinterpret_cast<void **>(&A), sizeof(uint8_t) * M_GLOBAL * K_GLOBAL));
checkCudaErrors(
cudaMalloc(reinterpret_cast<void **>(&B), sizeof(uint8_t) * N_GLOBAL * K_GLOBAL));
checkCudaErrors(cudaMalloc(reinterpret_cast<void **>(&C), sizeof(int) * M_GLOBAL * N_GLOBAL));
checkCudaErrors(cudaMalloc(reinterpret_cast<void **>(&D), sizeof(int) * M_GLOBAL * N_GLOBAL));
assert(((unsigned long long)A) % 128 == 0);
assert(((unsigned long long)B) % 128 == 0);
assert(((unsigned long long)C) % 128 == 0);
assert(((unsigned long long)D) % 128 == 0);
init_host_matrices(A_h, B_h, C_h);
checkCudaErrors(cudaMemcpy(A, A_h, sizeof(uint8_t) * M_GLOBAL * K_GLOBAL,
cudaMemcpyHostToDevice));
checkCudaErrors(cudaMemcpy(B, B_h, sizeof(uint8_t) * N_GLOBAL * K_GLOBAL,
cudaMemcpyHostToDevice));
checkCudaErrors(cudaMemcpy(C, C_h, sizeof(int) * M_GLOBAL * N_GLOBAL,
cudaMemcpyHostToDevice));
checkCudaErrors(cudaMemset(D, 0, sizeof(int) * M_GLOBAL * N_GLOBAL));
printf("Preparing data for GPU...\n");
assert(((unsigned long long)A) % 128 == 0);
assert(((unsigned long long)B) % 128 == 0);
assert(((unsigned long long)C) % 128 == 0);
assert(((unsigned long long)D) % 128 == 0);
enum {
// Compute the right amount of shared memory to request.
// We need shared memory to hold per-CTA C and D matrix tiles, and to cache
// per-CTA chunks
// of the A and B matrices. Therefore, the right amount to request is the
// maximum of those
// two numbers.
SHMEM_SZ = MAX(sizeof(uint8_t) * (BLOCK_COL_TILES * M) *
(CHUNK_K * K + SKEW_UINT8) * 2,
M * (BLOCK_ROW_WARPS * WARP_ROW_TILES) * N *
(BLOCK_COL_WARPS * WARP_COL_TILES) * sizeof(int))
};
printf("Required shared memory size: %lu Kb\n", SHMEM_SZ / 1024UL);
int alpha = 1;
int beta = 1;
cudaEvent_t start, stop;
checkCudaErrors(cudaEventCreate(&start));
checkCudaErrors(cudaEventCreate(&stop));
checkCudaErrors(cudaEventRecord(start));
// If enough shared memory available on the GPU use high performant kernel
if (deviceProp.sharedMemPerMultiprocessor >= SHMEM_SZ) {
printf("Computing... using high performance kernel compute_gemm_imma \n");
checkCudaErrors(cudaFuncSetAttribute(
compute_gemm_imma, cudaFuncAttributeMaxDynamicSharedMemorySize,
SHMEM_SZ));
checkKernelErrors(
(compute_gemm_imma<<<deviceProp.multiProcessorCount, THREADS_PER_BLOCK,
SHMEM_SZ>>>(A, B, C, D, alpha, beta)));
#if CPU_DEBUG
checkCudaErrors(cudaMemcpy(result_hD, D, sizeof(int) * M_GLOBAL * N_GLOBAL,
cudaMemcpyDeviceToHost));
#endif
} else {
dim3 gridDim;
dim3 blockDim;
// blockDim.x must be a multiple of warpSize
// 128x4 means we have 16 warps and a block computes a 64x64 output tile
blockDim.x = 128;
blockDim.y = 4;
gridDim.x = (M_GLOBAL + (WMMA_M * blockDim.x / 32 - 1)) /
(WMMA_M * blockDim.x / 32);
gridDim.y = (N_GLOBAL + WMMA_N * blockDim.y - 1) / (WMMA_N * blockDim.y);
printf("Computing... using simple_wmma_gemm_imma kernel\n");
simple_wmma_gemm_imma<<<gridDim, blockDim>>>(A, B, C, D, M_GLOBAL, N_GLOBAL,
K_GLOBAL, alpha, beta);
#if CPU_DEBUG
checkCudaErrors(cudaMemcpy(result_hD, D, sizeof(int) * M_GLOBAL * N_GLOBAL,
cudaMemcpyDeviceToHost));
#endif
}
checkCudaErrors(cudaEventRecord(stop));
checkCudaErrors(cudaEventSynchronize(stop));
#if CPU_DEBUG
printf("Verifying correctness of the computations...\n");
memcpy(result_host, C_h, sizeof(int) * M_GLOBAL * N_GLOBAL);
matMultiplyOnHost(A_h, B_h, result_host, alpha, beta, M_GLOBAL, K_GLOBAL,
K_GLOBAL, N_GLOBAL, M_GLOBAL, N_GLOBAL);
for (int i = 0; i < N_GLOBAL * M_GLOBAL; i++) {
if (abs(result_hD[i] - result_host[i]) > 0) {
printf("mismatch i=%d result_hD=%d result_host=%d\n", i, result_hD[i],
result_host[i]);
// Tensor cores require a GPU of Volta (SM72) architecture or higher.
if (deviceProp.major < 7 || (deviceProp.major <= 7 && deviceProp.minor < 2)) {
printf("immaTensorCoreGemm requires SM 7.2 or higher to use Tensor Cores. "
"Exiting...\n");
exit(EXIT_WAIVED);
}
}
free(result_host);
free(result_hD);
printf("M: %d (%d x %d)\n", M_GLOBAL, M, M_TILES);
printf("N: %d (%d x %d)\n", N_GLOBAL, N, N_TILES);
printf("K: %d (%d x %d)\n", K_GLOBAL, K, K_TILES);
uint8_t *A_h = NULL;
uint8_t *B_h = NULL;
int *C_h = NULL;
#if CPU_DEBUG
int *result_hD = NULL;
int *result_host = NULL;
#endif
float milliseconds = 0;
A_h = (uint8_t *)malloc(sizeof(uint8_t) * M_GLOBAL * K_GLOBAL);
B_h = (uint8_t *)malloc(sizeof(uint8_t) * K_GLOBAL * N_GLOBAL);
C_h = (int *)malloc(sizeof(int) * M_GLOBAL * N_GLOBAL);
#if CPU_DEBUG
result_hD = (int *)malloc(sizeof(int) * M_GLOBAL * N_GLOBAL);
result_host = (int *)malloc(sizeof(int) * M_GLOBAL * N_GLOBAL);
#endif
checkCudaErrors(cudaEventElapsedTime(&milliseconds, start, stop));
uint8_t *A = NULL;
uint8_t *B = NULL;
int *C = NULL;
int *D = NULL;
checkCudaErrors(cudaMalloc(reinterpret_cast<void **>(&A), sizeof(uint8_t) * M_GLOBAL * K_GLOBAL));
checkCudaErrors(cudaMalloc(reinterpret_cast<void **>(&B), sizeof(uint8_t) * N_GLOBAL * K_GLOBAL));
checkCudaErrors(cudaMalloc(reinterpret_cast<void **>(&C), sizeof(int) * M_GLOBAL * N_GLOBAL));
checkCudaErrors(cudaMalloc(reinterpret_cast<void **>(&D), sizeof(int) * M_GLOBAL * N_GLOBAL));
assert(((unsigned long long)A) % 128 == 0);
assert(((unsigned long long)B) % 128 == 0);
assert(((unsigned long long)C) % 128 == 0);
assert(((unsigned long long)D) % 128 == 0);
init_host_matrices(A_h, B_h, C_h);
checkCudaErrors(cudaMemcpy(A, A_h, sizeof(uint8_t) * M_GLOBAL * K_GLOBAL, cudaMemcpyHostToDevice));
checkCudaErrors(cudaMemcpy(B, B_h, sizeof(uint8_t) * N_GLOBAL * K_GLOBAL, cudaMemcpyHostToDevice));
checkCudaErrors(cudaMemcpy(C, C_h, sizeof(int) * M_GLOBAL * N_GLOBAL, cudaMemcpyHostToDevice));
checkCudaErrors(cudaMemset(D, 0, sizeof(int) * M_GLOBAL * N_GLOBAL));
printf("Preparing data for GPU...\n");
assert(((unsigned long long)A) % 128 == 0);
assert(((unsigned long long)B) % 128 == 0);
assert(((unsigned long long)C) % 128 == 0);
assert(((unsigned long long)D) % 128 == 0);
enum {
// Compute the right amount of shared memory to request.
// We need shared memory to hold per-CTA C and D matrix tiles, and to cache
// per-CTA chunks
// of the A and B matrices. Therefore, the right amount to request is the
// maximum of those
// two numbers.
SHMEM_SZ = MAX(sizeof(uint8_t) * (BLOCK_COL_TILES * M) * (CHUNK_K * K + SKEW_UINT8) * 2,
M * (BLOCK_ROW_WARPS * WARP_ROW_TILES) * N * (BLOCK_COL_WARPS * WARP_COL_TILES) * sizeof(int))
};
printf("Required shared memory size: %lu Kb\n", SHMEM_SZ / 1024UL);
int alpha = 1;
int beta = 1;
cudaEvent_t start, stop;
checkCudaErrors(cudaEventCreate(&start));
checkCudaErrors(cudaEventCreate(&stop));
checkCudaErrors(cudaEventRecord(start));
// If enough shared memory available on the GPU use high performant kernel
if (deviceProp.sharedMemPerMultiprocessor >= SHMEM_SZ) {
printf("Computing... using high performance kernel compute_gemm_imma \n");
checkCudaErrors(cudaFuncSetAttribute(compute_gemm_imma, cudaFuncAttributeMaxDynamicSharedMemorySize, SHMEM_SZ));
checkKernelErrors((compute_gemm_imma<<<deviceProp.multiProcessorCount, THREADS_PER_BLOCK, SHMEM_SZ>>>(
A, B, C, D, alpha, beta)));
#if CPU_DEBUG
checkCudaErrors(cudaMemcpy(result_hD, D, sizeof(int) * M_GLOBAL * N_GLOBAL, cudaMemcpyDeviceToHost));
#endif
}
else {
dim3 gridDim;
dim3 blockDim;
// blockDim.x must be a multiple of warpSize
// 128x4 means we have 16 warps and a block computes a 64x64 output tile
blockDim.x = 128;
blockDim.y = 4;
gridDim.x = (M_GLOBAL + (WMMA_M * blockDim.x / 32 - 1)) / (WMMA_M * blockDim.x / 32);
gridDim.y = (N_GLOBAL + WMMA_N * blockDim.y - 1) / (WMMA_N * blockDim.y);
printf("Computing... using simple_wmma_gemm_imma kernel\n");
simple_wmma_gemm_imma<<<gridDim, blockDim>>>(A, B, C, D, M_GLOBAL, N_GLOBAL, K_GLOBAL, alpha, beta);
#if CPU_DEBUG
checkCudaErrors(cudaMemcpy(result_hD, D, sizeof(int) * M_GLOBAL * N_GLOBAL, cudaMemcpyDeviceToHost));
#endif
}
checkCudaErrors(cudaEventRecord(stop));
checkCudaErrors(cudaEventSynchronize(stop));
#if CPU_DEBUG
printf("Verifying correctness of the computations...\n");
memcpy(result_host, C_h, sizeof(int) * M_GLOBAL * N_GLOBAL);
matMultiplyOnHost(A_h, B_h, result_host, alpha, beta, M_GLOBAL, K_GLOBAL, K_GLOBAL, N_GLOBAL, M_GLOBAL, N_GLOBAL);
for (int i = 0; i < N_GLOBAL * M_GLOBAL; i++) {
if (abs(result_hD[i] - result_host[i]) > 0) {
printf("mismatch i=%d result_hD=%d result_host=%d\n", i, result_hD[i], result_host[i]);
}
}
free(result_host);
free(result_hD);
#endif
float milliseconds = 0;
checkCudaErrors(cudaEventElapsedTime(&milliseconds, start, stop));
printf("Time: %f ms\n", milliseconds);
printf("TOPS: %.2f\n", (((double)M_GLOBAL * N_GLOBAL * K_GLOBAL * 2)/(milliseconds/1000.)) / 1e12);
printf("TOPS: %.2f\n", (((double)M_GLOBAL * N_GLOBAL * K_GLOBAL * 2) / (milliseconds / 1000.)) / 1e12);
free(A_h);
free(B_h);
free(C_h);
checkCudaErrors(cudaFree(reinterpret_cast<void *>(A)));
checkCudaErrors(cudaFree(reinterpret_cast<void *>(B)));
checkCudaErrors(cudaFree(reinterpret_cast<void *>(C)));
checkCudaErrors(cudaFree(reinterpret_cast<void *>(D)));
free(A_h);
free(B_h);
free(C_h);
checkCudaErrors(cudaFree(reinterpret_cast<void *>(A)));
checkCudaErrors(cudaFree(reinterpret_cast<void *>(B)));
checkCudaErrors(cudaFree(reinterpret_cast<void *>(C)));
checkCudaErrors(cudaFree(reinterpret_cast<void *>(D)));
return EXIT_SUCCESS;
return EXIT_SUCCESS;
}

620
Samples/3_CUDA_Features/jacobiCudaGraphs/jacobi.cu
View File

@@ -29,6 +29,7 @@
#include <cuda_runtime.h>
#include <helper_cuda.h>
#include <vector>
#include "jacobi.h"
namespace cg = cooperative_groups;
@@ -39,355 +40,356 @@ namespace cg = cooperative_groups;
#if !defined(__CUDA_ARCH__) || __CUDA_ARCH__ >= 600
#else
__device__ double atomicAdd(double *address, double val) {
unsigned long long int *address_as_ull = (unsigned long long int *)address;
unsigned long long int old = *address_as_ull, assumed;
__device__ double atomicAdd(double *address, double val)
{
unsigned long long int *address_as_ull = (unsigned long long int *)address;
unsigned long long int old = *address_as_ull, assumed;
do {
assumed = old;
old = atomicCAS(address_as_ull, assumed,
__double_as_longlong(val + __longlong_as_double(assumed)));
do {
assumed = old;
old = atomicCAS(address_as_ull, assumed, __double_as_longlong(val + __longlong_as_double(assumed)));
// Note: uses integer comparison to avoid hang in case of NaN (since NaN !=
// NaN)
} while (assumed != old);
// Note: uses integer comparison to avoid hang in case of NaN (since NaN !=
// NaN)
} while (assumed != old);
return __longlong_as_double(old);
return __longlong_as_double(old);
}
#endif
static __global__ void JacobiMethod(const float *A, const double *b,
const float conv_threshold, double *x,
double *x_new, double *sum) {
// Handle to thread block group
cg::thread_block cta = cg::this_thread_block();
__shared__ double x_shared[N_ROWS]; // N_ROWS == n
__shared__ double b_shared[ROWS_PER_CTA + 1];
static __global__ void
JacobiMethod(const float *A, const double *b, const float conv_threshold, double *x, double *x_new, double *sum)
{
// Handle to thread block group
cg::thread_block cta = cg::this_thread_block();
__shared__ double x_shared[N_ROWS]; // N_ROWS == n
__shared__ double b_shared[ROWS_PER_CTA + 1];
for (int i = threadIdx.x; i < N_ROWS; i += blockDim.x) {
x_shared[i] = x[i];
}
if (threadIdx.x < ROWS_PER_CTA) {
int k = threadIdx.x;
for (int i = k + (blockIdx.x * ROWS_PER_CTA);
(k < ROWS_PER_CTA) && (i < N_ROWS);
k += ROWS_PER_CTA, i += ROWS_PER_CTA) {
b_shared[i % (ROWS_PER_CTA + 1)] = b[i];
}
}
cg::sync(cta);
cg::thread_block_tile<32> tile32 = cg::tiled_partition<32>(cta);
for (int k = 0, i = blockIdx.x * ROWS_PER_CTA;
(k < ROWS_PER_CTA) && (i < N_ROWS); k++, i++) {
double rowThreadSum = 0.0;
for (int j = threadIdx.x; j < N_ROWS; j += blockDim.x) {
rowThreadSum += (A[i * N_ROWS + j] * x_shared[j]);
for (int i = threadIdx.x; i < N_ROWS; i += blockDim.x) {
x_shared[i] = x[i];
}
for (int offset = tile32.size() / 2; offset > 0; offset /= 2) {
rowThreadSum += tile32.shfl_down(rowThreadSum, offset);
if (threadIdx.x < ROWS_PER_CTA) {
int k = threadIdx.x;
for (int i = k + (blockIdx.x * ROWS_PER_CTA); (k < ROWS_PER_CTA) && (i < N_ROWS);
k += ROWS_PER_CTA, i += ROWS_PER_CTA) {
b_shared[i % (ROWS_PER_CTA + 1)] = b[i];
}
}
if (tile32.thread_rank() == 0) {
atomicAdd(&b_shared[i % (ROWS_PER_CTA + 1)], -rowThreadSum);
}
}
cg::sync(cta);
cg::sync(cta);
cg::thread_block_tile<32> tile32 = cg::tiled_partition<32>(cta);
if (threadIdx.x < ROWS_PER_CTA) {
cg::thread_block_tile<ROWS_PER_CTA> tile8 =
cg::tiled_partition<ROWS_PER_CTA>(cta);
double temp_sum = 0.0;
for (int k = 0, i = blockIdx.x * ROWS_PER_CTA; (k < ROWS_PER_CTA) && (i < N_ROWS); k++, i++) {
double rowThreadSum = 0.0;
for (int j = threadIdx.x; j < N_ROWS; j += blockDim.x) {
rowThreadSum += (A[i * N_ROWS + j] * x_shared[j]);
}
int k = threadIdx.x;
for (int offset = tile32.size() / 2; offset > 0; offset /= 2) {
rowThreadSum += tile32.shfl_down(rowThreadSum, offset);
}
for (int i = k + (blockIdx.x * ROWS_PER_CTA);
(k < ROWS_PER_CTA) && (i < N_ROWS);
k += ROWS_PER_CTA, i += ROWS_PER_CTA) {
double dx = b_shared[i % (ROWS_PER_CTA + 1)];
dx /= A[i * N_ROWS + i];
x_new[i] = (x_shared[i] + dx);
temp_sum += fabs(dx);
if (tile32.thread_rank() == 0) {
atomicAdd(&b_shared[i % (ROWS_PER_CTA + 1)], -rowThreadSum);
}
}
for (int offset = tile8.size() / 2; offset > 0; offset /= 2) {
temp_sum += tile8.shfl_down(temp_sum, offset);
}
cg::sync(cta);
if (tile8.thread_rank() == 0) {
atomicAdd(sum, temp_sum);
if (threadIdx.x < ROWS_PER_CTA) {
cg::thread_block_tile<ROWS_PER_CTA> tile8 = cg::tiled_partition<ROWS_PER_CTA>(cta);
double temp_sum = 0.0;
int k = threadIdx.x;
for (int i = k + (blockIdx.x * ROWS_PER_CTA); (k < ROWS_PER_CTA) && (i < N_ROWS);
k += ROWS_PER_CTA, i += ROWS_PER_CTA) {
double dx = b_shared[i % (ROWS_PER_CTA + 1)];
dx /= A[i * N_ROWS + i];
x_new[i] = (x_shared[i] + dx);
temp_sum += fabs(dx);
}
for (int offset = tile8.size() / 2; offset > 0; offset /= 2) {
temp_sum += tile8.shfl_down(temp_sum, offset);
}
if (tile8.thread_rank() == 0) {
atomicAdd(sum, temp_sum);
}
}
}
}
// Thread block size for finalError kernel should be multiple of 32
static __global__ void finalError(double *x, double *g_sum) {
// Handle to thread block group
cg::thread_block cta = cg::this_thread_block();
extern __shared__ double warpSum[];
double sum = 0.0;
static __global__ void finalError(double *x, double *g_sum)
{
// Handle to thread block group
cg::thread_block cta = cg::this_thread_block();
extern __shared__ double warpSum[];
double sum = 0.0;
int globalThreadId = blockIdx.x * blockDim.x + threadIdx.x;
int globalThreadId = blockIdx.x * blockDim.x + threadIdx.x;
for (int i = globalThreadId; i < N_ROWS; i += blockDim.x * gridDim.x) {
double d = x[i] - 1.0;
sum += fabs(d);
}
for (int i = globalThreadId; i < N_ROWS; i += blockDim.x * gridDim.x) {
double d = x[i] - 1.0;
sum += fabs(d);
}
cg::thread_block_tile<32> tile32 = cg::tiled_partition<32>(cta);
cg::thread_block_tile<32> tile32 = cg::tiled_partition<32>(cta);
for (int offset = tile32.size() / 2; offset > 0; offset /= 2) {
sum += tile32.shfl_down(sum, offset);
}
if (tile32.thread_rank() == 0) {
warpSum[threadIdx.x / warpSize] = sum;
}
cg::sync(cta);
double blockSum = 0.0;
if (threadIdx.x < (blockDim.x / warpSize)) {
blockSum = warpSum[threadIdx.x];
}
if (threadIdx.x < 32) {
for (int offset = tile32.size() / 2; offset > 0; offset /= 2) {
blockSum += tile32.shfl_down(blockSum, offset);
sum += tile32.shfl_down(sum, offset);
}
if (tile32.thread_rank() == 0) {
atomicAdd(g_sum, blockSum);
warpSum[threadIdx.x / warpSize] = sum;
}
}
}
double JacobiMethodGpuCudaGraphExecKernelSetParams(
const float *A, const double *b, const float conv_threshold,
const int max_iter, double *x, double *x_new, cudaStream_t stream) {
// CTA size
dim3 nthreads(256, 1, 1);
// grid size
dim3 nblocks((N_ROWS / ROWS_PER_CTA) + 2, 1, 1);
cudaGraph_t graph;
cudaGraphExec_t graphExec = NULL;
cg::sync(cta);
double sum = 0.0;
double *d_sum = NULL;
checkCudaErrors(cudaMalloc(&d_sum, sizeof(double)));
std::vector<cudaGraphNode_t> nodeDependencies;
cudaGraphNode_t memcpyNode, jacobiKernelNode, memsetNode;
cudaMemcpy3DParms memcpyParams = {0};
cudaMemsetParams memsetParams = {0};
memsetParams.dst = (void *)d_sum;
memsetParams.value = 0;
memsetParams.pitch = 0;
// elementSize can be max 4 bytes, so we take sizeof(float) and width=2
memsetParams.elementSize = sizeof(float);
memsetParams.width = 2;
memsetParams.height = 1;
checkCudaErrors(cudaGraphCreate(&graph, 0));
checkCudaErrors(
cudaGraphAddMemsetNode(&memsetNode, graph, NULL, 0, &memsetParams));
nodeDependencies.push_back(memsetNode);
cudaKernelNodeParams NodeParams0, NodeParams1;
NodeParams0.func = (void *)JacobiMethod;
NodeParams0.gridDim = nblocks;
NodeParams0.blockDim = nthreads;
NodeParams0.sharedMemBytes = 0;
void *kernelArgs0[6] = {(void *)&A, (void *)&b, (void *)&conv_threshold,
(void *)&x, (void *)&x_new, (void *)&d_sum};
NodeParams0.kernelParams = kernelArgs0;
NodeParams0.extra = NULL;
checkCudaErrors(
cudaGraphAddKernelNode(&jacobiKernelNode, graph, nodeDependencies.data(),
nodeDependencies.size(), &NodeParams0));
nodeDependencies.clear();
nodeDependencies.push_back(jacobiKernelNode);
memcpyParams.srcArray = NULL;
memcpyParams.srcPos = make_cudaPos(0, 0, 0);
memcpyParams.srcPtr = make_cudaPitchedPtr(d_sum, sizeof(double), 1, 1);
memcpyParams.dstArray = NULL;
memcpyParams.dstPos = make_cudaPos(0, 0, 0);
memcpyParams.dstPtr = make_cudaPitchedPtr(&sum, sizeof(double), 1, 1);
memcpyParams.extent = make_cudaExtent(sizeof(double), 1, 1);
memcpyParams.kind = cudaMemcpyDeviceToHost;
checkCudaErrors(
cudaGraphAddMemcpyNode(&memcpyNode, graph, nodeDependencies.data(),
nodeDependencies.size(), &memcpyParams));
checkCudaErrors(cudaGraphInstantiate(&graphExec, graph, NULL, NULL, 0));
NodeParams1.func = (void *)JacobiMethod;
NodeParams1.gridDim = nblocks;
NodeParams1.blockDim = nthreads;
NodeParams1.sharedMemBytes = 0;
void *kernelArgs1[6] = {(void *)&A, (void *)&b, (void *)&conv_threshold,
(void *)&x_new, (void *)&x, (void *)&d_sum};
NodeParams1.kernelParams = kernelArgs1;
NodeParams1.extra = NULL;
int k = 0;
for (k = 0; k < max_iter; k++) {
checkCudaErrors(cudaGraphExecKernelNodeSetParams(
graphExec, jacobiKernelNode,
((k & 1) == 0) ? &NodeParams0 : &NodeParams1));
checkCudaErrors(cudaGraphLaunch(graphExec, stream));
checkCudaErrors(cudaStreamSynchronize(stream));
if (sum <= conv_threshold) {
checkCudaErrors(cudaMemsetAsync(d_sum, 0, sizeof(double), stream));
nblocks.x = (N_ROWS / nthreads.x) + 1;
size_t sharedMemSize = ((nthreads.x / 32) + 1) * sizeof(double);
if ((k & 1) == 0) {
finalError<<<nblocks, nthreads, sharedMemSize, stream>>>(x_new, d_sum);
} else {
finalError<<<nblocks, nthreads, sharedMemSize, stream>>>(x, d_sum);
}
checkCudaErrors(cudaMemcpyAsync(&sum, d_sum, sizeof(double),
cudaMemcpyDeviceToHost, stream));
checkCudaErrors(cudaStreamSynchronize(stream));
printf("GPU iterations : %d\n", k + 1);
printf("GPU error : %.3e\n", sum);
break;
double blockSum = 0.0;
if (threadIdx.x < (blockDim.x / warpSize)) {
blockSum = warpSum[threadIdx.x];
}
}
checkCudaErrors(cudaFree(d_sum));
return sum;
}
double JacobiMethodGpuCudaGraphExecUpdate(const float *A, const double *b,
const float conv_threshold,
const int max_iter, double *x,
double *x_new, cudaStream_t stream) {
// CTA size
dim3 nthreads(256, 1, 1);
// grid size
dim3 nblocks((N_ROWS / ROWS_PER_CTA) + 2, 1, 1);
cudaGraph_t graph;
cudaGraphExec_t graphExec = NULL;
double sum = 0.0;
double *d_sum;
checkCudaErrors(cudaMalloc(&d_sum, sizeof(double)));
int k = 0;
for (k = 0; k < max_iter; k++) {
checkCudaErrors(
cudaStreamBeginCapture(stream, cudaStreamCaptureModeGlobal));
checkCudaErrors(cudaMemsetAsync(d_sum, 0, sizeof(double), stream));
if ((k & 1) == 0) {
JacobiMethod<<<nblocks, nthreads, 0, stream>>>(A, b, conv_threshold, x,
x_new, d_sum);
} else {
JacobiMethod<<<nblocks, nthreads, 0, stream>>>(A, b, conv_threshold,
x_new, x, d_sum);
}
checkCudaErrors(cudaMemcpyAsync(&sum, d_sum, sizeof(double),
cudaMemcpyDeviceToHost, stream));
checkCudaErrors(cudaStreamEndCapture(stream, &graph));
if (graphExec == NULL) {
checkCudaErrors(cudaGraphInstantiate(&graphExec, graph, NULL, NULL, 0));
} else {
cudaGraphExecUpdateResult updateResult_out;
checkCudaErrors(
cudaGraphExecUpdate(graphExec, graph, NULL, &updateResult_out));
if (updateResult_out != cudaGraphExecUpdateSuccess) {
if (graphExec != NULL) {
checkCudaErrors(cudaGraphExecDestroy(graphExec));
if (threadIdx.x < 32) {
for (int offset = tile32.size() / 2; offset > 0; offset /= 2) {
blockSum += tile32.shfl_down(blockSum, offset);
}
if (tile32.thread_rank() == 0) {
atomicAdd(g_sum, blockSum);
}
printf("k = %d graph update failed with error - %d\n", k,
updateResult_out);
checkCudaErrors(cudaGraphInstantiate(&graphExec, graph, NULL, NULL, 0));
}
}
checkCudaErrors(cudaGraphLaunch(graphExec, stream));
checkCudaErrors(cudaStreamSynchronize(stream));
if (sum <= conv_threshold) {
checkCudaErrors(cudaMemsetAsync(d_sum, 0, sizeof(double), stream));
nblocks.x = (N_ROWS / nthreads.x) + 1;
size_t sharedMemSize = ((nthreads.x / 32) + 1) * sizeof(double);
if ((k & 1) == 0) {
finalError<<<nblocks, nthreads, sharedMemSize, stream>>>(x_new, d_sum);
} else {
finalError<<<nblocks, nthreads, sharedMemSize, stream>>>(x, d_sum);
}
checkCudaErrors(cudaMemcpyAsync(&sum, d_sum, sizeof(double),
cudaMemcpyDeviceToHost, stream));
checkCudaErrors(cudaStreamSynchronize(stream));
printf("GPU iterations : %d\n", k + 1);
printf("GPU error : %.3e\n", sum);
break;
}
}
checkCudaErrors(cudaFree(d_sum));
return sum;
}
double JacobiMethodGpu(const float *A, const double *b,
const float conv_threshold, const int max_iter,
double *x, double *x_new, cudaStream_t stream) {
// CTA size
dim3 nthreads(256, 1, 1);
// grid size
dim3 nblocks((N_ROWS / ROWS_PER_CTA) + 2, 1, 1);
double JacobiMethodGpuCudaGraphExecKernelSetParams(const float *A,
const double *b,
const float conv_threshold,
const int max_iter,
double *x,
double *x_new,
cudaStream_t stream)
{
// CTA size
dim3 nthreads(256, 1, 1);
// grid size
dim3 nblocks((N_ROWS / ROWS_PER_CTA) + 2, 1, 1);
cudaGraph_t graph;
cudaGraphExec_t graphExec = NULL;
double sum = 0.0;
double *d_sum;
checkCudaErrors(cudaMalloc(&d_sum, sizeof(double)));
int k = 0;
double sum = 0.0;
double *d_sum = NULL;
checkCudaErrors(cudaMalloc(&d_sum, sizeof(double)));
for (k = 0; k < max_iter; k++) {
checkCudaErrors(cudaMemsetAsync(d_sum, 0, sizeof(double), stream));
if ((k & 1) == 0) {
JacobiMethod<<<nblocks, nthreads, 0, stream>>>(A, b, conv_threshold, x,
x_new, d_sum);
} else {
JacobiMethod<<<nblocks, nthreads, 0, stream>>>(A, b, conv_threshold,
x_new, x, d_sum);
std::vector<cudaGraphNode_t> nodeDependencies;
cudaGraphNode_t memcpyNode, jacobiKernelNode, memsetNode;
cudaMemcpy3DParms memcpyParams = {0};
cudaMemsetParams memsetParams = {0};
memsetParams.dst = (void *)d_sum;
memsetParams.value = 0;
memsetParams.pitch = 0;
// elementSize can be max 4 bytes, so we take sizeof(float) and width=2
memsetParams.elementSize = sizeof(float);
memsetParams.width = 2;
memsetParams.height = 1;
checkCudaErrors(cudaGraphCreate(&graph, 0));
checkCudaErrors(cudaGraphAddMemsetNode(&memsetNode, graph, NULL, 0, &memsetParams));
nodeDependencies.push_back(memsetNode);
cudaKernelNodeParams NodeParams0, NodeParams1;
NodeParams0.func = (void *)JacobiMethod;
NodeParams0.gridDim = nblocks;
NodeParams0.blockDim = nthreads;
NodeParams0.sharedMemBytes = 0;
void *kernelArgs0[6] = {
(void *)&A, (void *)&b, (void *)&conv_threshold, (void *)&x, (void *)&x_new, (void *)&d_sum};
NodeParams0.kernelParams = kernelArgs0;
NodeParams0.extra = NULL;
checkCudaErrors(cudaGraphAddKernelNode(
&jacobiKernelNode, graph, nodeDependencies.data(), nodeDependencies.size(), &NodeParams0));
nodeDependencies.clear();
nodeDependencies.push_back(jacobiKernelNode);
memcpyParams.srcArray = NULL;
memcpyParams.srcPos = make_cudaPos(0, 0, 0);
memcpyParams.srcPtr = make_cudaPitchedPtr(d_sum, sizeof(double), 1, 1);
memcpyParams.dstArray = NULL;
memcpyParams.dstPos = make_cudaPos(0, 0, 0);
memcpyParams.dstPtr = make_cudaPitchedPtr(&sum, sizeof(double), 1, 1);
memcpyParams.extent = make_cudaExtent(sizeof(double), 1, 1);
memcpyParams.kind = cudaMemcpyDeviceToHost;
checkCudaErrors(
cudaGraphAddMemcpyNode(&memcpyNode, graph, nodeDependencies.data(), nodeDependencies.size(), &memcpyParams));
checkCudaErrors(cudaGraphInstantiate(&graphExec, graph, NULL, NULL, 0));
NodeParams1.func = (void *)JacobiMethod;
NodeParams1.gridDim = nblocks;
NodeParams1.blockDim = nthreads;
NodeParams1.sharedMemBytes = 0;
void *kernelArgs1[6] = {
(void *)&A, (void *)&b, (void *)&conv_threshold, (void *)&x_new, (void *)&x, (void *)&d_sum};
NodeParams1.kernelParams = kernelArgs1;
NodeParams1.extra = NULL;
int k = 0;
for (k = 0; k < max_iter; k++) {
checkCudaErrors(cudaGraphExecKernelNodeSetParams(
graphExec, jacobiKernelNode, ((k & 1) == 0) ? &NodeParams0 : &NodeParams1));
checkCudaErrors(cudaGraphLaunch(graphExec, stream));
checkCudaErrors(cudaStreamSynchronize(stream));
if (sum <= conv_threshold) {
checkCudaErrors(cudaMemsetAsync(d_sum, 0, sizeof(double), stream));
nblocks.x = (N_ROWS / nthreads.x) + 1;
size_t sharedMemSize = ((nthreads.x / 32) + 1) * sizeof(double);
if ((k & 1) == 0) {
finalError<<<nblocks, nthreads, sharedMemSize, stream>>>(x_new, d_sum);
}
else {
finalError<<<nblocks, nthreads, sharedMemSize, stream>>>(x, d_sum);
}
checkCudaErrors(cudaMemcpyAsync(&sum, d_sum, sizeof(double), cudaMemcpyDeviceToHost, stream));
checkCudaErrors(cudaStreamSynchronize(stream));
printf("GPU iterations : %d\n", k + 1);
printf("GPU error : %.3e\n", sum);
break;
}
}
checkCudaErrors(cudaMemcpyAsync(&sum, d_sum, sizeof(double),
cudaMemcpyDeviceToHost, stream));
checkCudaErrors(cudaStreamSynchronize(stream));
if (sum <= conv_threshold) {
checkCudaErrors(cudaMemsetAsync(d_sum, 0, sizeof(double), stream));
nblocks.x = (N_ROWS / nthreads.x) + 1;
size_t sharedMemSize = ((nthreads.x / 32) + 1) * sizeof(double);
if ((k & 1) == 0) {
finalError<<<nblocks, nthreads, sharedMemSize, stream>>>(x_new, d_sum);
} else {
finalError<<<nblocks, nthreads, sharedMemSize, stream>>>(x, d_sum);
}
checkCudaErrors(cudaMemcpyAsync(&sum, d_sum, sizeof(double),
cudaMemcpyDeviceToHost, stream));
checkCudaErrors(cudaStreamSynchronize(stream));
printf("GPU iterations : %d\n", k + 1);
printf("GPU error : %.3e\n", sum);
break;
}
}
checkCudaErrors(cudaFree(d_sum));
return sum;
checkCudaErrors(cudaFree(d_sum));
return sum;
}
double JacobiMethodGpuCudaGraphExecUpdate(const float *A,
const double *b,
const float conv_threshold,
const int max_iter,
double *x,
double *x_new,
cudaStream_t stream)
{
// CTA size
dim3 nthreads(256, 1, 1);
// grid size
dim3 nblocks((N_ROWS / ROWS_PER_CTA) + 2, 1, 1);
cudaGraph_t graph;
cudaGraphExec_t graphExec = NULL;
double sum = 0.0;
double *d_sum;
checkCudaErrors(cudaMalloc(&d_sum, sizeof(double)));
int k = 0;
for (k = 0; k < max_iter; k++) {
checkCudaErrors(cudaStreamBeginCapture(stream, cudaStreamCaptureModeGlobal));
checkCudaErrors(cudaMemsetAsync(d_sum, 0, sizeof(double), stream));
if ((k & 1) == 0) {
JacobiMethod<<<nblocks, nthreads, 0, stream>>>(A, b, conv_threshold, x, x_new, d_sum);
}
else {
JacobiMethod<<<nblocks, nthreads, 0, stream>>>(A, b, conv_threshold, x_new, x, d_sum);
}
checkCudaErrors(cudaMemcpyAsync(&sum, d_sum, sizeof(double), cudaMemcpyDeviceToHost, stream));
checkCudaErrors(cudaStreamEndCapture(stream, &graph));
if (graphExec == NULL) {
checkCudaErrors(cudaGraphInstantiate(&graphExec, graph, NULL, NULL, 0));
}
else {
cudaGraphExecUpdateResult updateResult_out;
checkCudaErrors(cudaGraphExecUpdate(graphExec, graph, NULL, &updateResult_out));
if (updateResult_out != cudaGraphExecUpdateSuccess) {
if (graphExec != NULL) {
checkCudaErrors(cudaGraphExecDestroy(graphExec));
}
printf("k = %d graph update failed with error - %d\n", k, updateResult_out);
checkCudaErrors(cudaGraphInstantiate(&graphExec, graph, NULL, NULL, 0));
}
}
checkCudaErrors(cudaGraphLaunch(graphExec, stream));
checkCudaErrors(cudaStreamSynchronize(stream));
if (sum <= conv_threshold) {
checkCudaErrors(cudaMemsetAsync(d_sum, 0, sizeof(double), stream));
nblocks.x = (N_ROWS / nthreads.x) + 1;
size_t sharedMemSize = ((nthreads.x / 32) + 1) * sizeof(double);
if ((k & 1) == 0) {
finalError<<<nblocks, nthreads, sharedMemSize, stream>>>(x_new, d_sum);
}
else {
finalError<<<nblocks, nthreads, sharedMemSize, stream>>>(x, d_sum);
}
checkCudaErrors(cudaMemcpyAsync(&sum, d_sum, sizeof(double), cudaMemcpyDeviceToHost, stream));
checkCudaErrors(cudaStreamSynchronize(stream));
printf("GPU iterations : %d\n", k + 1);
printf("GPU error : %.3e\n", sum);
break;
}
}
checkCudaErrors(cudaFree(d_sum));
return sum;
}
double JacobiMethodGpu(const float *A,
const double *b,
const float conv_threshold,
const int max_iter,
double *x,
double *x_new,
cudaStream_t stream)
{
// CTA size
dim3 nthreads(256, 1, 1);
// grid size
dim3 nblocks((N_ROWS / ROWS_PER_CTA) + 2, 1, 1);
double sum = 0.0;
double *d_sum;
checkCudaErrors(cudaMalloc(&d_sum, sizeof(double)));
int k = 0;
for (k = 0; k < max_iter; k++) {
checkCudaErrors(cudaMemsetAsync(d_sum, 0, sizeof(double), stream));
if ((k & 1) == 0) {
JacobiMethod<<<nblocks, nthreads, 0, stream>>>(A, b, conv_threshold, x, x_new, d_sum);
}
else {
JacobiMethod<<<nblocks, nthreads, 0, stream>>>(A, b, conv_threshold, x_new, x, d_sum);
}
checkCudaErrors(cudaMemcpyAsync(&sum, d_sum, sizeof(double), cudaMemcpyDeviceToHost, stream));
checkCudaErrors(cudaStreamSynchronize(stream));
if (sum <= conv_threshold) {
checkCudaErrors(cudaMemsetAsync(d_sum, 0, sizeof(double), stream));
nblocks.x = (N_ROWS / nthreads.x) + 1;
size_t sharedMemSize = ((nthreads.x / 32) + 1) * sizeof(double);
if ((k & 1) == 0) {
finalError<<<nblocks, nthreads, sharedMemSize, stream>>>(x_new, d_sum);
}
else {
finalError<<<nblocks, nthreads, sharedMemSize, stream>>>(x, d_sum);
}
checkCudaErrors(cudaMemcpyAsync(&sum, d_sum, sizeof(double), cudaMemcpyDeviceToHost, stream));
checkCudaErrors(cudaStreamSynchronize(stream));
printf("GPU iterations : %d\n", k + 1);
printf("GPU error : %.3e\n", sum);
break;
}
}
checkCudaErrors(cudaFree(d_sum));
return sum;
}

2
Samples/3_CUDA_Features/jacobiCudaGraphs/jacobi.h
View File

@@ -30,4 +30,4 @@
#define N_ROWS 512
#endif
#endif

285
Samples/3_CUDA_Features/jacobiCudaGraphs/main.cpp
View File

@@ -43,24 +43,37 @@
#include <math.h>
#include <stdio.h>
#include <stdlib.h>
#include "jacobi.h"
// Run the Jacobi method for A*x = b on GPU with CUDA Graph -
// cudaGraphExecKernelNodeSetParams().
extern double JacobiMethodGpuCudaGraphExecKernelSetParams(
const float *A, const double *b, const float conv_threshold,
const int max_iter, double *x, double *x_new, cudaStream_t stream);
extern double JacobiMethodGpuCudaGraphExecKernelSetParams(const float *A,
const double *b,
const float conv_threshold,
const int max_iter,
double *x,
double *x_new,
cudaStream_t stream);
// Run the Jacobi method for A*x = b on GPU with Instantiated CUDA Graph Update
// API - cudaGraphExecUpdate().
extern double JacobiMethodGpuCudaGraphExecUpdate(
const float *A, const double *b, const float conv_threshold,
const int max_iter, double *x, double *x_new, cudaStream_t stream);
extern double JacobiMethodGpuCudaGraphExecUpdate(const float *A,
const double *b,
const float conv_threshold,
const int max_iter,
double *x,
double *x_new,
cudaStream_t stream);
// Run the Jacobi method for A*x = b on GPU without CUDA Graph.
extern double JacobiMethodGpu(const float *A, const double *b,
const float conv_threshold, const int max_iter,
double *x, double *x_new, cudaStream_t stream);
extern double JacobiMethodGpu(const float *A,
const double *b,
const float conv_threshold,
const int max_iter,
double *x,
double *x_new,
cudaStream_t stream);
// creates N_ROWS x N_ROWS matrix A with N_ROWS+1 on the diagonal and 1
// elsewhere. The elements of the right hand side b all equal 2*n, hence the
@@ -68,148 +81,146 @@ extern double JacobiMethodGpu(const float *A, const double *b,
void createLinearSystem(float *A, double *b);
// Run the Jacobi method for A*x = b on CPU.
void JacobiMethodCPU(float *A, double *b, float conv_threshold, int max_iter,
int *numit, double *x);
void JacobiMethodCPU(float *A, double *b, float conv_threshold, int max_iter, int *numit, double *x);
int main(int argc, char **argv) {
if (checkCmdLineFlag(argc, (const char **)argv, "help")) {
printf("Command line: jacobiCudaGraphs [-option]\n");
printf("Valid options:\n");
printf(
"-gpumethod=<0,1 or 2> : 0 - [Default] "
"JacobiMethodGpuCudaGraphExecKernelSetParams\n");
printf(" : 1 - JacobiMethodGpuCudaGraphExecUpdate\n");
printf(" : 2 - JacobiMethodGpu - Non CUDA Graph\n");
printf("-device=device_num : cuda device id");
printf("-help : Output a help message\n");
exit(EXIT_SUCCESS);
}
int gpumethod = 0;
if (checkCmdLineFlag(argc, (const char **)argv, "gpumethod")) {
gpumethod = getCmdLineArgumentInt(argc, (const char **)argv, "gpumethod");
if (gpumethod < 0 || gpumethod > 2) {
printf("Error: gpumethod must be 0 or 1 or 2, gpumethod=%d is invalid\n",
gpumethod);
exit(EXIT_SUCCESS);
int main(int argc, char **argv)
{
if (checkCmdLineFlag(argc, (const char **)argv, "help")) {
printf("Command line: jacobiCudaGraphs [-option]\n");
printf("Valid options:\n");
printf("-gpumethod=<0,1 or 2> : 0 - [Default] "
"JacobiMethodGpuCudaGraphExecKernelSetParams\n");
printf(" : 1 - JacobiMethodGpuCudaGraphExecUpdate\n");
printf(" : 2 - JacobiMethodGpu - Non CUDA Graph\n");
printf("-device=device_num : cuda device id");
printf("-help : Output a help message\n");
exit(EXIT_SUCCESS);
}
}
int dev = findCudaDevice(argc, (const char **)argv);
int gpumethod = 0;
if (checkCmdLineFlag(argc, (const char **)argv, "gpumethod")) {
gpumethod = getCmdLineArgumentInt(argc, (const char **)argv, "gpumethod");
double *b = NULL;
float *A = NULL;
checkCudaErrors(cudaMallocHost(&b, N_ROWS * sizeof(double)));
memset(b, 0, N_ROWS * sizeof(double));
checkCudaErrors(cudaMallocHost(&A, N_ROWS * N_ROWS * sizeof(float)));
memset(A, 0, N_ROWS * N_ROWS * sizeof(float));
if (gpumethod < 0 || gpumethod > 2) {
printf("Error: gpumethod must be 0 or 1 or 2, gpumethod=%d is invalid\n", gpumethod);
exit(EXIT_SUCCESS);
}
}
createLinearSystem(A, b);
double *x = NULL;
// start with array of all zeroes
x = (double *)calloc(N_ROWS, sizeof(double));
int dev = findCudaDevice(argc, (const char **)argv);
float conv_threshold = 1.0e-2;
int max_iter = 4 * N_ROWS * N_ROWS;
int cnt = 0;
double *b = NULL;
float *A = NULL;
checkCudaErrors(cudaMallocHost(&b, N_ROWS * sizeof(double)));
memset(b, 0, N_ROWS * sizeof(double));
checkCudaErrors(cudaMallocHost(&A, N_ROWS * N_ROWS * sizeof(float)));
memset(A, 0, N_ROWS * N_ROWS * sizeof(float));
// create timer
StopWatchInterface *timerCPU = NULL, *timerGpu = NULL;
sdkCreateTimer(&timerCPU);
createLinearSystem(A, b);
double *x = NULL;
// start with array of all zeroes
x = (double *)calloc(N_ROWS, sizeof(double));
sdkStartTimer(&timerCPU);
JacobiMethodCPU(A, b, conv_threshold, max_iter, &cnt, x);
float conv_threshold = 1.0e-2;
int max_iter = 4 * N_ROWS * N_ROWS;
int cnt = 0;
double sum = 0.0;
// Compute error
for (int i = 0; i < N_ROWS; i++) {
double d = x[i] - 1.0;
sum += fabs(d);
}
sdkStopTimer(&timerCPU);
printf("CPU iterations : %d\n", cnt);
printf("CPU error : %.3e\n", sum);
printf("CPU Processing time: %f (ms)\n", sdkGetTimerValue(&timerCPU));
// create timer
StopWatchInterface *timerCPU = NULL, *timerGpu = NULL;
sdkCreateTimer(&timerCPU);
float *d_A;
double *d_b, *d_x, *d_x_new;
cudaStream_t stream1;
checkCudaErrors(cudaStreamCreateWithFlags(&stream1, cudaStreamNonBlocking));
checkCudaErrors(cudaMalloc(&d_b, sizeof(double) * N_ROWS));
checkCudaErrors(cudaMalloc(&d_A, sizeof(float) * N_ROWS * N_ROWS));
checkCudaErrors(cudaMalloc(&d_x, sizeof(double) * N_ROWS));
checkCudaErrors(cudaMalloc(&d_x_new, sizeof(double) * N_ROWS));
sdkStartTimer(&timerCPU);
JacobiMethodCPU(A, b, conv_threshold, max_iter, &cnt, x);
checkCudaErrors(cudaMemsetAsync(d_x, 0, sizeof(double) * N_ROWS, stream1));
checkCudaErrors(
cudaMemsetAsync(d_x_new, 0, sizeof(double) * N_ROWS, stream1));
checkCudaErrors(cudaMemcpyAsync(d_A, A, sizeof(float) * N_ROWS * N_ROWS,
cudaMemcpyHostToDevice, stream1));
checkCudaErrors(cudaMemcpyAsync(d_b, b, sizeof(double) * N_ROWS,
cudaMemcpyHostToDevice, stream1));
sdkCreateTimer(&timerGpu);
sdkStartTimer(&timerGpu);
double sumGPU = 0.0;
if (gpumethod == 0) {
sumGPU = JacobiMethodGpuCudaGraphExecKernelSetParams(
d_A, d_b, conv_threshold, max_iter, d_x, d_x_new, stream1);
} else if (gpumethod == 1) {
sumGPU = JacobiMethodGpuCudaGraphExecUpdate(
d_A, d_b, conv_threshold, max_iter, d_x, d_x_new, stream1);
} else if (gpumethod == 2) {
sumGPU = JacobiMethodGpu(d_A, d_b, conv_threshold, max_iter, d_x, d_x_new,
stream1);
}
sdkStopTimer(&timerGpu);
printf("GPU Processing time: %f (ms)\n", sdkGetTimerValue(&timerGpu));
checkCudaErrors(cudaFree(d_b));
checkCudaErrors(cudaFree(d_A));
checkCudaErrors(cudaFree(d_x));
checkCudaErrors(cudaFree(d_x_new));
checkCudaErrors(cudaFreeHost(A));
checkCudaErrors(cudaFreeHost(b));
printf("&&&& jacobiCudaGraphs %s\n",
(fabs(sum - sumGPU) < conv_threshold) ? "PASSED" : "FAILED");
return (fabs(sum - sumGPU) < conv_threshold) ? EXIT_SUCCESS : EXIT_FAILURE;
}
void createLinearSystem(float *A, double *b) {
int i, j;
for (i = 0; i < N_ROWS; i++) {
b[i] = 2.0 * N_ROWS;
for (j = 0; j < N_ROWS; j++) A[i * N_ROWS + j] = 1.0;
A[i * N_ROWS + i] = N_ROWS + 1.0;
}
}
void JacobiMethodCPU(float *A, double *b, float conv_threshold, int max_iter,
int *num_iter, double *x) {
double *x_new;
x_new = (double *)calloc(N_ROWS, sizeof(double));
int k;
for (k = 0; k < max_iter; k++) {
double sum = 0.0;
// Compute error
for (int i = 0; i < N_ROWS; i++) {
double temp_dx = b[i];
for (int j = 0; j < N_ROWS; j++) temp_dx -= A[i * N_ROWS + j] * x[j];
temp_dx /= A[i * N_ROWS + i];
x_new[i] += temp_dx;
sum += fabs(temp_dx);
double d = x[i] - 1.0;
sum += fabs(d);
}
sdkStopTimer(&timerCPU);
printf("CPU iterations : %d\n", cnt);
printf("CPU error : %.3e\n", sum);
printf("CPU Processing time: %f (ms)\n", sdkGetTimerValue(&timerCPU));
float *d_A;
double *d_b, *d_x, *d_x_new;
cudaStream_t stream1;
checkCudaErrors(cudaStreamCreateWithFlags(&stream1, cudaStreamNonBlocking));
checkCudaErrors(cudaMalloc(&d_b, sizeof(double) * N_ROWS));
checkCudaErrors(cudaMalloc(&d_A, sizeof(float) * N_ROWS * N_ROWS));
checkCudaErrors(cudaMalloc(&d_x, sizeof(double) * N_ROWS));
checkCudaErrors(cudaMalloc(&d_x_new, sizeof(double) * N_ROWS));
checkCudaErrors(cudaMemsetAsync(d_x, 0, sizeof(double) * N_ROWS, stream1));
checkCudaErrors(cudaMemsetAsync(d_x_new, 0, sizeof(double) * N_ROWS, stream1));
checkCudaErrors(cudaMemcpyAsync(d_A, A, sizeof(float) * N_ROWS * N_ROWS, cudaMemcpyHostToDevice, stream1));
checkCudaErrors(cudaMemcpyAsync(d_b, b, sizeof(double) * N_ROWS, cudaMemcpyHostToDevice, stream1));
sdkCreateTimer(&timerGpu);
sdkStartTimer(&timerGpu);
double sumGPU = 0.0;
if (gpumethod == 0) {
sumGPU = JacobiMethodGpuCudaGraphExecKernelSetParams(d_A, d_b, conv_threshold, max_iter, d_x, d_x_new, stream1);
}
else if (gpumethod == 1) {
sumGPU = JacobiMethodGpuCudaGraphExecUpdate(d_A, d_b, conv_threshold, max_iter, d_x, d_x_new, stream1);
}
else if (gpumethod == 2) {
sumGPU = JacobiMethodGpu(d_A, d_b, conv_threshold, max_iter, d_x, d_x_new, stream1);
}
for (int i = 0; i < N_ROWS; i++) x[i] = x_new[i];
sdkStopTimer(&timerGpu);
printf("GPU Processing time: %f (ms)\n", sdkGetTimerValue(&timerGpu));
if (sum <= conv_threshold) break;
}
*num_iter = k + 1;
free(x_new);
checkCudaErrors(cudaFree(d_b));
checkCudaErrors(cudaFree(d_A));
checkCudaErrors(cudaFree(d_x));
checkCudaErrors(cudaFree(d_x_new));
checkCudaErrors(cudaFreeHost(A));
checkCudaErrors(cudaFreeHost(b));
printf("&&&& jacobiCudaGraphs %s\n", (fabs(sum - sumGPU) < conv_threshold) ? "PASSED" : "FAILED");
return (fabs(sum - sumGPU) < conv_threshold) ? EXIT_SUCCESS : EXIT_FAILURE;
}
void createLinearSystem(float *A, double *b)
{
int i, j;
for (i = 0; i < N_ROWS; i++) {
b[i] = 2.0 * N_ROWS;
for (j = 0; j < N_ROWS; j++)
A[i * N_ROWS + j] = 1.0;
A[i * N_ROWS + i] = N_ROWS + 1.0;
}
}
void JacobiMethodCPU(float *A, double *b, float conv_threshold, int max_iter, int *num_iter, double *x)
{
double *x_new;
x_new = (double *)calloc(N_ROWS, sizeof(double));
int k;
for (k = 0; k < max_iter; k++) {
double sum = 0.0;
for (int i = 0; i < N_ROWS; i++) {
double temp_dx = b[i];
for (int j = 0; j < N_ROWS; j++)
temp_dx -= A[i * N_ROWS + j] * x[j];
temp_dx /= A[i * N_ROWS + i];
x_new[i] += temp_dx;
sum += fabs(temp_dx);
}
for (int i = 0; i < N_ROWS; i++)
x[i] = x_new[i];
if (sum <= conv_threshold)
break;
}
*num_iter = k + 1;
free(x_new);
}

948
Samples/3_CUDA_Features/memMapIPCDrv/memMapIpc.cpp
View File

File diff suppressed because it is too large Load Diff

100
Samples/3_CUDA_Features/newdelete/container.hpp
View File

@@ -31,16 +31,16 @@
//
////////////////////////////////////////////////////////////////////////////////
template <class T>
class Container {
public:
__device__ Container() { ; }
template <class T> class Container
{
public:
__device__ Container() { ; }
__device__ virtual ~Container() { ; }
__device__ virtual ~Container() { ; }
__device__ virtual void push(T e) = 0;
__device__ virtual void push(T e) = 0;
__device__ virtual bool pop(T& e) = 0;
__device__ virtual bool pop(T &e) = 0;
};
////////////////////////////////////////////////////////////////////////////////
@@ -52,47 +52,57 @@ class Container {
//
////////////////////////////////////////////////////////////////////////////////
template <class T>
class Vector : public Container<T> {
public:
// Constructor, data is allocated on the heap
// NOTE: This must be called from only one thread
__device__ Vector(int max_size) : m_top(-1) { m_data = new T[max_size]; }
// Constructor, data uses preallocated buffer via placement new
__device__ Vector(int max_size, T* preallocated_buffer) : m_top(-1) {
m_data = new (preallocated_buffer) T[max_size];
}
// Destructor, data is freed
// NOTE: This must be called from only one thread
__device__ ~Vector() {
if (m_data) delete[] m_data;
}
__device__ virtual void push(T e) {
if (m_data) {
// Atomically increment the top idx
int idx = atomicAdd(&(this->m_top), 1);
m_data[idx + 1] = e;
template <class T> class Vector : public Container<T>
{
public:
// Constructor, data is allocated on the heap
// NOTE: This must be called from only one thread
__device__ Vector(int max_size)
: m_top(-1)
{
m_data = new T[max_size];
}
}
__device__ virtual bool pop(T& e) {
if (m_data && m_top >= 0) {
// Atomically decrement the top idx
int idx = atomicAdd(&(this->m_top), -1);
if (idx >= 0) {
e = m_data[idx];
return true;
}
// Constructor, data uses preallocated buffer via placement new
__device__ Vector(int max_size, T *preallocated_buffer)
: m_top(-1)
{
m_data = new (preallocated_buffer) T[max_size];
}
return false;
}
private:
int m_size;
T* m_data;
// Destructor, data is freed
// NOTE: This must be called from only one thread
__device__ ~Vector()
{
if (m_data)
delete[] m_data;
}
int m_top;
__device__ virtual void push(T e)
{
if (m_data) {
// Atomically increment the top idx
int idx = atomicAdd(&(this->m_top), 1);
m_data[idx + 1] = e;
}
}
__device__ virtual bool pop(T &e)
{
if (m_data && m_top >= 0) {
// Atomically decrement the top idx
int idx = atomicAdd(&(this->m_top), -1);
if (idx >= 0) {
e = m_data[idx];
return true;
}
}
return false;
}
private:
int m_size;
T *m_data;
int m_top;
};

336
Samples/3_CUDA_Features/newdelete/newdelete.cu
View File

@@ -29,17 +29,14 @@
// new and delete operators and virtual function declarations available with
// CUDA 4.0.
#include <cooperative_groups.h>
#include <stdio.h>
#include <cooperative_groups.h>
namespace cg = cooperative_groups;
#include <helper_cuda.h>
#include <stdlib.h>
#include <vector>
#include <algorithm>
#include <helper_cuda.h>
#include <stdlib.h>
#include <vector>
const char *sSDKsample = "newdelete";
@@ -51,12 +48,13 @@ const char *sSDKsample = "newdelete";
//
////////////////////////////////////////////////////////////////////////////////
__global__ void vectorCreate(Container<int> **g_container, int max_size) {
// The Vector object and the data storage are allocated in device heap memory.
// This makes it persistent for the lifetime of the CUDA context.
// The grid has only one thread as only a single object instance is needed.
__global__ void vectorCreate(Container<int> **g_container, int max_size)
{
// The Vector object and the data storage are allocated in device heap memory.
// This makes it persistent for the lifetime of the CUDA context.
// The grid has only one thread as only a single object instance is needed.
*g_container = new Vector<int>(max_size);
*g_container = new Vector<int>(max_size);
}
////////////////////////////////////////////////////////////////////////////////
@@ -65,26 +63,29 @@ __global__ void vectorCreate(Container<int> **g_container, int max_size) {
//
////////////////////////////////////////////////////////////////////////////////
__global__ void containerFill(Container<int> **g_container) {
// All threads of the grid cooperatively populate the shared Container object
// with data.
if (threadIdx.x == 0) {
(*g_container)->push(blockIdx.x);
}
__global__ void containerFill(Container<int> **g_container)
{
// All threads of the grid cooperatively populate the shared Container object
// with data.
if (threadIdx.x == 0) {
(*g_container)->push(blockIdx.x);
}
}
__global__ void containerConsume(Container<int> **g_container, int *d_result) {
// All threads of the grid cooperatively consume the data from the shared
// Container object.
int idx = blockIdx.x * blockDim.x + threadIdx.x;
__global__ void containerConsume(Container<int> **g_container, int *d_result)
{
// All threads of the grid cooperatively consume the data from the shared
// Container object.
int idx = blockIdx.x * blockDim.x + threadIdx.x;
int v;
int v;
if ((*g_container)->pop(v)) {
d_result[idx] = v;
} else {
d_result[idx] = -1;
}
if ((*g_container)->pop(v)) {
d_result[idx] = v;
}
else {
d_result[idx] = -1;
}
}
////////////////////////////////////////////////////////////////////////////////
@@ -93,9 +94,7 @@ __global__ void containerConsume(Container<int> **g_container, int *d_result) {
//
////////////////////////////////////////////////////////////////////////////////
__global__ void containerDelete(Container<int> **g_container) {
delete *g_container;
}
__global__ void containerDelete(Container<int> **g_container) { delete *g_container; }
////////////////////////////////////////////////////////////////////////////////
//
@@ -104,87 +103,92 @@ __global__ void containerDelete(Container<int> **g_container) {
//
////////////////////////////////////////////////////////////////////////////////
__global__ void placementNew(int *d_result) {
// Handle to thread block group
cg::thread_block cta = cg::this_thread_block();
__shared__ unsigned char __align__(8) s_buffer[sizeof(Vector<int>)];
__shared__ int __align__(8) s_data[1024];
__shared__ Vector<int> *s_vector;
__global__ void placementNew(int *d_result)
{
// Handle to thread block group
cg::thread_block cta = cg::this_thread_block();
__shared__ unsigned char __align__(8) s_buffer[sizeof(Vector<int>)];
__shared__ int __align__(8) s_data[1024];
__shared__ Vector<int> *s_vector;
// The first thread of the block initializes the shared Vector object.
// The placement new operator enables the Vector object and the data array top
// be placed in shared memory.
if (threadIdx.x == 0) {
s_vector = new (s_buffer) Vector<int>(1024, s_data);
}
// The first thread of the block initializes the shared Vector object.
// The placement new operator enables the Vector object and the data array top
// be placed in shared memory.
if (threadIdx.x == 0) {
s_vector = new (s_buffer) Vector<int>(1024, s_data);
}
cg::sync(cta);
cg::sync(cta);
if ((threadIdx.x & 1) == 0) {
s_vector->push(threadIdx.x >> 1);
}
if ((threadIdx.x & 1) == 0) {
s_vector->push(threadIdx.x >> 1);
}
// Need to sync as the vector implementation does not support concurrent
// push/pop operations.
cg::sync(cta);
// Need to sync as the vector implementation does not support concurrent
// push/pop operations.
cg::sync(cta);
int v;
int v;
if (s_vector->pop(v)) {
d_result[threadIdx.x] = v;
} else {
d_result[threadIdx.x] = -1;
}
if (s_vector->pop(v)) {
d_result[threadIdx.x] = v;
}
else {
d_result[threadIdx.x] = -1;
}
// Note: deleting objects placed in shared memory is not necessary (lifetime
// of shared memory is that of the block)
// Note: deleting objects placed in shared memory is not necessary (lifetime
// of shared memory is that of the block)
}
struct ComplexType_t {
int a;
int b;
float c;
float d;
struct ComplexType_t
{
int a;
int b;
float c;
float d;
};
__global__ void complexVector(int *d_result) {
// Handle to thread block group
cg::thread_block cta = cg::this_thread_block();
__shared__ unsigned char __align__(8) s_buffer[sizeof(Vector<ComplexType_t>)];
__shared__ ComplexType_t __align__(8) s_data[1024];
__shared__ Vector<ComplexType_t> *s_vector;
__global__ void complexVector(int *d_result)
{
// Handle to thread block group
cg::thread_block cta = cg::this_thread_block();
__shared__ unsigned char __align__(8) s_buffer[sizeof(Vector<ComplexType_t>)];
__shared__ ComplexType_t __align__(8) s_data[1024];
__shared__ Vector<ComplexType_t> *s_vector;
// The first thread of the block initializes the shared Vector object.
// The placement new operator enables the Vector object and the data array top
// be placed in shared memory.
if (threadIdx.x == 0) {
s_vector = new (s_buffer) Vector<ComplexType_t>(1024, s_data);
}
// The first thread of the block initializes the shared Vector object.
// The placement new operator enables the Vector object and the data array top
// be placed in shared memory.
if (threadIdx.x == 0) {
s_vector = new (s_buffer) Vector<ComplexType_t>(1024, s_data);
}
cg::sync(cta);
cg::sync(cta);
if ((threadIdx.x & 1) == 0) {
ComplexType_t data;
data.a = threadIdx.x >> 1;
data.b = blockIdx.x;
data.c = threadIdx.x / (float)(blockDim.x);
data.d = blockIdx.x / (float)(gridDim.x);
if ((threadIdx.x & 1) == 0) {
ComplexType_t data;
data.a = threadIdx.x >> 1;
data.b = blockIdx.x;
data.c = threadIdx.x / (float)(blockDim.x);
data.d = blockIdx.x / (float)(gridDim.x);
s_vector->push(data);
}
s_vector->push(data);
}
cg::sync(cta);
cg::sync(cta);
ComplexType_t v;
ComplexType_t v;
if (s_vector->pop(v)) {
d_result[threadIdx.x] = v.a;
} else {
d_result[threadIdx.x] = -1;
}
if (s_vector->pop(v)) {
d_result[threadIdx.x] = v.a;
}
else {
d_result[threadIdx.x] = -1;
}
// Note: deleting objects placed in shared memory is not necessary (lifetime
// of shared memory is that of the block)
// Note: deleting objects placed in shared memory is not necessary (lifetime
// of shared memory is that of the block)
}
////////////////////////////////////////////////////////////////////////////////
@@ -193,74 +197,77 @@ __global__ void complexVector(int *d_result) {
//
////////////////////////////////////////////////////////////////////////////////
bool checkResult(int *d_result, int N) {
std::vector<int> h_result;
h_result.resize(N);
bool checkResult(int *d_result, int N)
{
std::vector<int> h_result;
h_result.resize(N);
checkCudaErrors(cudaMemcpy(&h_result[0], d_result, N * sizeof(int),
cudaMemcpyDeviceToHost));
std::sort(h_result.begin(), h_result.end());
checkCudaErrors(cudaMemcpy(&h_result[0], d_result, N * sizeof(int), cudaMemcpyDeviceToHost));
std::sort(h_result.begin(), h_result.end());
bool success = true;
bool test = false;
bool success = true;
bool test = false;
int value = 0;
int value = 0;
for (int i = 0; i < N; ++i) {
if (h_result[i] != -1) {
test = true;
for (int i = 0; i < N; ++i) {
if (h_result[i] != -1) {
test = true;
}
if (test && (value++) != h_result[i]) {
success = false;
}
}
if (test && (value++) != h_result[i]) {
success = false;
}
}
return success;
return success;
}
bool testContainer(Container<int> **d_container, int blocks, int threads) {
int *d_result;
cudaMalloc(&d_result, blocks * threads * sizeof(int));
bool testContainer(Container<int> **d_container, int blocks, int threads)
{
int *d_result;
cudaMalloc(&d_result, blocks * threads * sizeof(int));
containerFill<<<blocks, threads>>>(d_container);
containerConsume<<<blocks, threads>>>(d_container, d_result);
containerDelete<<<1, 1>>>(d_container);
checkCudaErrors(cudaDeviceSynchronize());
containerFill<<<blocks, threads>>>(d_container);
containerConsume<<<blocks, threads>>>(d_container, d_result);
containerDelete<<<1, 1>>>(d_container);
checkCudaErrors(cudaDeviceSynchronize());
bool success = checkResult(d_result, blocks * threads);
bool success = checkResult(d_result, blocks * threads);
cudaFree(d_result);
cudaFree(d_result);
return success;
return success;
}
bool testPlacementNew(int threads) {
int *d_result;
cudaMalloc(&d_result, threads * sizeof(int));
bool testPlacementNew(int threads)
{
int *d_result;
cudaMalloc(&d_result, threads * sizeof(int));
placementNew<<<1, threads>>>(d_result);
checkCudaErrors(cudaDeviceSynchronize());
placementNew<<<1, threads>>>(d_result);
checkCudaErrors(cudaDeviceSynchronize());
bool success = checkResult(d_result, threads);
bool success = checkResult(d_result, threads);
cudaFree(d_result);
cudaFree(d_result);
return success;
return success;
}
bool testComplexType(int threads) {
int *d_result;
cudaMalloc(&d_result, threads * sizeof(int));
bool testComplexType(int threads)
{
int *d_result;
cudaMalloc(&d_result, threads * sizeof(int));
complexVector<<<1, threads>>>(d_result);
checkCudaErrors(cudaDeviceSynchronize());
complexVector<<<1, threads>>>(d_result);
checkCudaErrors(cudaDeviceSynchronize());
bool success = checkResult(d_result, threads);
bool success = checkResult(d_result, threads);
cudaFree(d_result);
cudaFree(d_result);
return success;
return success;
}
////////////////////////////////////////////////////////////////////////////////
@@ -269,41 +276,42 @@ bool testComplexType(int threads) {
//
////////////////////////////////////////////////////////////////////////////////
int main(int argc, char **argv) {
printf("%s Starting...\n\n", sSDKsample);
int main(int argc, char **argv)
{
printf("%s Starting...\n\n", sSDKsample);
// use command-line specified CUDA device, otherwise use device with highest
// Gflops/s
findCudaDevice(argc, (const char **)argv);
// use command-line specified CUDA device, otherwise use device with highest
// Gflops/s
findCudaDevice(argc, (const char **)argv);
// set the heap size for device size new/delete to 128 MB
checkCudaErrors(cudaDeviceSetLimit(cudaLimitMallocHeapSize, 128 * (1 << 20)));
// set the heap size for device size new/delete to 128 MB
checkCudaErrors(cudaDeviceSetLimit(cudaLimitMallocHeapSize, 128 * (1 << 20)));
Container<int> **d_container;
checkCudaErrors(cudaMalloc(&d_container, sizeof(Container<int> **)));
Container<int> **d_container;
checkCudaErrors(cudaMalloc(&d_container, sizeof(Container<int> **)));
bool bTest = false;
int test_passed = 0;
bool bTest = false;
int test_passed = 0;
printf(" > Container = Vector test ");
vectorCreate<<<1, 1>>>(d_container, 128 * 128);
bTest = testContainer(d_container, 128, 128);
printf(bTest ? "OK\n\n" : "NOT OK\n\n");
test_passed += (bTest ? 1 : 0);
printf(" > Container = Vector test ");
vectorCreate<<<1, 1>>>(d_container, 128 * 128);
bTest = testContainer(d_container, 128, 128);
printf(bTest ? "OK\n\n" : "NOT OK\n\n");
test_passed += (bTest ? 1 : 0);
checkCudaErrors(cudaFree(d_container));
checkCudaErrors(cudaFree(d_container));
printf(" > Container = Vector, using placement new on SMEM buffer test ");
bTest = testPlacementNew(1024);
printf(bTest ? "OK\n\n" : "NOT OK\n\n");
test_passed += (bTest ? 1 : 0);
printf(" > Container = Vector, using placement new on SMEM buffer test ");
bTest = testPlacementNew(1024);
printf(bTest ? "OK\n\n" : "NOT OK\n\n");
test_passed += (bTest ? 1 : 0);
printf(" > Container = Vector, with user defined datatype test ");
bTest = testComplexType(1024);
printf(bTest ? "OK\n\n" : "NOT OK\n\n");
test_passed += (bTest ? 1 : 0);
printf(" > Container = Vector, with user defined datatype test ");
bTest = testComplexType(1024);
printf(bTest ? "OK\n\n" : "NOT OK\n\n");
test_passed += (bTest ? 1 : 0);
printf("Test Summary: %d/3 succesfully run\n", test_passed);
printf("Test Summary: %d/3 succesfully run\n", test_passed);
exit(test_passed == 3 ? EXIT_SUCCESS : EXIT_FAILURE);
exit(test_passed == 3 ? EXIT_SUCCESS : EXIT_FAILURE);
}

364
Samples/3_CUDA_Features/ptxjit/ptxjit.cpp
View File

@@ -34,10 +34,10 @@
*/
// System includes
#include <iostream>
#include <math.h>
#include <stdio.h>
#include <string.h>
#include <iostream>
// CUDA driver & runtime
#include <cuda.h>
@@ -47,7 +47,7 @@
#define CUDA_DRIVER_API
#include <helper_cuda.h>
#include <helper_cuda_drvapi.h>
#include <helper_functions.h> // helper for shared that are common to CUDA Samples
#include <helper_functions.h> // helper for shared that are common to CUDA Samples
#if defined(_WIN64) || defined(__LP64__)
#define PTX_FILE "ptxjit_kernel64.ptx"
@@ -57,189 +57,191 @@
const char *sSDKname = "PTX Just In Time (JIT) Compilation (no-qatest)";
bool inline findModulePath(const char *module_file, std::string &module_path,
char **argv, std::string &ptx_source) {
char *actual_path = sdkFindFilePath(module_file, argv[0]);
bool inline findModulePath(const char *module_file, std::string &module_path, char **argv, std::string &ptx_source)
{
char *actual_path = sdkFindFilePath(module_file, argv[0]);
if (actual_path) {
module_path = actual_path;
} else {
printf("> findModulePath file not found: <%s> \n", module_file);
return false;
}
if (module_path.empty()) {
printf("> findModulePath file not found: <%s> \n", module_file);
return false;
} else {
printf("> findModulePath <%s>\n", module_path.c_str());
if (module_path.rfind(".ptx") != std::string::npos) {
FILE *fp = fopen(module_path.c_str(), "rb");
fseek(fp, 0, SEEK_END);
int file_size = ftell(fp);
char *buf = new char[file_size + 1];
fseek(fp, 0, SEEK_SET);
fread(buf, sizeof(char), file_size, fp);
fclose(fp);
buf[file_size] = '\0';
ptx_source = buf;
delete[] buf;
if (actual_path) {
module_path = actual_path;
}
else {
printf("> findModulePath file not found: <%s> \n", module_file);
return false;
}
return true;
}
}
void ptxJIT(int argc, char **argv, CUmodule *phModule, CUfunction *phKernel,
CUlinkState *lState) {
CUjit_option options[6];
void *optionVals[6];
float walltime;
char error_log[8192], info_log[8192];
unsigned int logSize = 8192;
void *cuOut;
size_t outSize;
int myErr = 0;
std::string module_path, ptx_source;
// Setup linker options
// Return walltime from JIT compilation
options[0] = CU_JIT_WALL_TIME;
optionVals[0] = (void *)&walltime;
// Pass a buffer for info messages
options[1] = CU_JIT_INFO_LOG_BUFFER;
optionVals[1] = (void *)info_log;
// Pass the size of the info buffer
options[2] = CU_JIT_INFO_LOG_BUFFER_SIZE_BYTES;
optionVals[2] = (void *)(long)logSize;
// Pass a buffer for error message
options[3] = CU_JIT_ERROR_LOG_BUFFER;
optionVals[3] = (void *)error_log;
// Pass the size of the error buffer
options[4] = CU_JIT_ERROR_LOG_BUFFER_SIZE_BYTES;
optionVals[4] = (void *)(long)logSize;
// Make the linker verbose
options[5] = CU_JIT_LOG_VERBOSE;
optionVals[5] = (void *)1;
// Create a pending linker invocation
checkCudaErrors(cuLinkCreate(6, options, optionVals, lState));
// first search for the module path before we load the results
if (!findModulePath(PTX_FILE, module_path, argv, ptx_source)) {
printf("> findModulePath could not find <ptxjit_kernel> ptx\n");
exit(EXIT_FAILURE);
} else {
printf("> initCUDA loading module: <%s>\n", module_path.c_str());
}
// Load the PTX from the ptx file
printf("Loading ptxjit_kernel[] program\n");
myErr = cuLinkAddData(*lState, CU_JIT_INPUT_PTX, (void *)ptx_source.c_str(),
strlen(ptx_source.c_str()) + 1, 0, 0, 0, 0);
if (myErr != CUDA_SUCCESS) {
// Errors will be put in error_log, per CU_JIT_ERROR_LOG_BUFFER option
// above.
fprintf(stderr, "PTX Linker Error:\n%s\n", error_log);
}
// Complete the linker step
checkCudaErrors(cuLinkComplete(*lState, &cuOut, &outSize));
// Linker walltime and info_log were requested in options above.
printf("CUDA Link Completed in %fms. Linker Output:\n%s\n", walltime,
info_log);
// Load resulting cuBin into module
checkCudaErrors(cuModuleLoadData(phModule, cuOut));
// Locate the kernel entry poin
checkCudaErrors(cuModuleGetFunction(phKernel, *phModule, "myKernel"));
// Destroy the linker invocation
checkCudaErrors(cuLinkDestroy(*lState));
}
int main(int argc, char **argv) {
const unsigned int nThreads = 256;
const unsigned int nBlocks = 64;
const size_t memSize = nThreads * nBlocks * sizeof(int);
CUmodule hModule = 0;
CUfunction hKernel = 0;
CUlinkState lState;
int *d_data = 0;
int *h_data = 0;
int cuda_device = 0;
printf("[%s] - Starting...\n", sSDKname);
CUdevice dev = findCudaDeviceDRV(argc, (const char **)argv);
int driverVersion;
cudaDriverGetVersion(&driverVersion);
if (driverVersion < CUDART_VERSION) {
printf(
"driverVersion = %d < CUDART_VERSION = %d \n"
"Enhanced compatibility is not supported for this sample.. waving "
"execution\n",
driverVersion, CUDART_VERSION);
exit(EXIT_WAIVED);
}
// Allocate memory on host and device (Runtime API)
// NOTE: The runtime API will create the GPU Context implicitly here
if ((h_data = (int *)malloc(memSize)) == NULL) {
std::cerr << "Could not allocate host memory" << std::endl;
exit(EXIT_FAILURE);
}
checkCudaErrors(cudaMalloc(&d_data, memSize));
// JIT Compile the Kernel from PTX and get the Handles (Driver API)
ptxJIT(argc, argv, &hModule, &hKernel, &lState);
// Set the kernel parameters (Driver API)
dim3 block(nThreads, 1, 1);
dim3 grid(nBlocks, 1, 1);
void *args[1] = {&d_data};
// Launch the kernel (Driver API_)
checkCudaErrors(cuLaunchKernel(hKernel, grid.x, grid.y, grid.z, block.x,
block.y, block.z, 0, NULL, args, NULL));
std::cout << "CUDA kernel launched" << std::endl;
// Copy the result back to the host
checkCudaErrors(cudaMemcpy(h_data, d_data, memSize, cudaMemcpyDeviceToHost));
// Check the result
bool dataGood = true;
for (unsigned int i = 0; dataGood && i < nBlocks * nThreads; i++) {
if (h_data[i] != (int)i) {
std::cerr << "Error at " << i << std::endl;
dataGood = false;
if (module_path.empty()) {
printf("> findModulePath file not found: <%s> \n", module_file);
return false;
}
}
else {
printf("> findModulePath <%s>\n", module_path.c_str());
// Cleanup
if (d_data) {
checkCudaErrors(cudaFree(d_data));
d_data = 0;
}
if (module_path.rfind(".ptx") != std::string::npos) {
FILE *fp = fopen(module_path.c_str(), "rb");
fseek(fp, 0, SEEK_END);
int file_size = ftell(fp);
char *buf = new char[file_size + 1];
fseek(fp, 0, SEEK_SET);
fread(buf, sizeof(char), file_size, fp);
fclose(fp);
buf[file_size] = '\0';
ptx_source = buf;
delete[] buf;
}
if (h_data) {
free(h_data);
h_data = 0;
}
if (hModule) {
checkCudaErrors(cuModuleUnload(hModule));
hModule = 0;
}
return dataGood ? EXIT_SUCCESS : EXIT_FAILURE;
return true;
}
}
void ptxJIT(int argc, char **argv, CUmodule *phModule, CUfunction *phKernel, CUlinkState *lState)
{
CUjit_option options[6];
void *optionVals[6];
float walltime;
char error_log[8192], info_log[8192];
unsigned int logSize = 8192;
void *cuOut;
size_t outSize;
int myErr = 0;
std::string module_path, ptx_source;
// Setup linker options
// Return walltime from JIT compilation
options[0] = CU_JIT_WALL_TIME;
optionVals[0] = (void *)&walltime;
// Pass a buffer for info messages
options[1] = CU_JIT_INFO_LOG_BUFFER;
optionVals[1] = (void *)info_log;
// Pass the size of the info buffer
options[2] = CU_JIT_INFO_LOG_BUFFER_SIZE_BYTES;
optionVals[2] = (void *)(long)logSize;
// Pass a buffer for error message
options[3] = CU_JIT_ERROR_LOG_BUFFER;
optionVals[3] = (void *)error_log;
// Pass the size of the error buffer
options[4] = CU_JIT_ERROR_LOG_BUFFER_SIZE_BYTES;
optionVals[4] = (void *)(long)logSize;
// Make the linker verbose
options[5] = CU_JIT_LOG_VERBOSE;
optionVals[5] = (void *)1;
// Create a pending linker invocation
checkCudaErrors(cuLinkCreate(6, options, optionVals, lState));
// first search for the module path before we load the results
if (!findModulePath(PTX_FILE, module_path, argv, ptx_source)) {
printf("> findModulePath could not find <ptxjit_kernel> ptx\n");
exit(EXIT_FAILURE);
}
else {
printf("> initCUDA loading module: <%s>\n", module_path.c_str());
}
// Load the PTX from the ptx file
printf("Loading ptxjit_kernel[] program\n");
myErr = cuLinkAddData(
*lState, CU_JIT_INPUT_PTX, (void *)ptx_source.c_str(), strlen(ptx_source.c_str()) + 1, 0, 0, 0, 0);
if (myErr != CUDA_SUCCESS) {
// Errors will be put in error_log, per CU_JIT_ERROR_LOG_BUFFER option
// above.
fprintf(stderr, "PTX Linker Error:\n%s\n", error_log);
}
// Complete the linker step
checkCudaErrors(cuLinkComplete(*lState, &cuOut, &outSize));
// Linker walltime and info_log were requested in options above.
printf("CUDA Link Completed in %fms. Linker Output:\n%s\n", walltime, info_log);
// Load resulting cuBin into module
checkCudaErrors(cuModuleLoadData(phModule, cuOut));
// Locate the kernel entry poin
checkCudaErrors(cuModuleGetFunction(phKernel, *phModule, "myKernel"));
// Destroy the linker invocation
checkCudaErrors(cuLinkDestroy(*lState));
}
int main(int argc, char **argv)
{
const unsigned int nThreads = 256;
const unsigned int nBlocks = 64;
const size_t memSize = nThreads * nBlocks * sizeof(int);
CUmodule hModule = 0;
CUfunction hKernel = 0;
CUlinkState lState;
int *d_data = 0;
int *h_data = 0;
int cuda_device = 0;
printf("[%s] - Starting...\n", sSDKname);
CUdevice dev = findCudaDeviceDRV(argc, (const char **)argv);
int driverVersion;
cudaDriverGetVersion(&driverVersion);
if (driverVersion < CUDART_VERSION) {
printf("driverVersion = %d < CUDART_VERSION = %d \n"
"Enhanced compatibility is not supported for this sample.. waving "
"execution\n",
driverVersion,
CUDART_VERSION);
exit(EXIT_WAIVED);
}
// Allocate memory on host and device (Runtime API)
// NOTE: The runtime API will create the GPU Context implicitly here
if ((h_data = (int *)malloc(memSize)) == NULL) {
std::cerr << "Could not allocate host memory" << std::endl;
exit(EXIT_FAILURE);
}
checkCudaErrors(cudaMalloc(&d_data, memSize));
// JIT Compile the Kernel from PTX and get the Handles (Driver API)
ptxJIT(argc, argv, &hModule, &hKernel, &lState);
// Set the kernel parameters (Driver API)
dim3 block(nThreads, 1, 1);
dim3 grid(nBlocks, 1, 1);
void *args[1] = {&d_data};
// Launch the kernel (Driver API_)
checkCudaErrors(cuLaunchKernel(hKernel, grid.x, grid.y, grid.z, block.x, block.y, block.z, 0, NULL, args, NULL));
std::cout << "CUDA kernel launched" << std::endl;
// Copy the result back to the host
checkCudaErrors(cudaMemcpy(h_data, d_data, memSize, cudaMemcpyDeviceToHost));
// Check the result
bool dataGood = true;
for (unsigned int i = 0; dataGood && i < nBlocks * nThreads; i++) {
if (h_data[i] != (int)i) {
std::cerr << "Error at " << i << std::endl;
dataGood = false;
}
}
// Cleanup
if (d_data) {
checkCudaErrors(cudaFree(d_data));
d_data = 0;
}
if (h_data) {
free(h_data);
h_data = 0;
}
if (hModule) {
checkCudaErrors(cuModuleUnload(hModule));
hModule = 0;
}
return dataGood ? EXIT_SUCCESS : EXIT_FAILURE;
}

7
Samples/3_CUDA_Features/ptxjit/ptxjit_kernel.cu
View File

@@ -29,7 +29,8 @@
* Simple kernel for ptxjit demonstration.
*
*/
extern "C" __global__ void myKernel(int *data) {
int tid = blockIdx.x * blockDim.x + threadIdx.x;
data[tid] = tid;
extern "C" __global__ void myKernel(int *data)
{
int tid = blockIdx.x * blockDim.x + threadIdx.x;
data[tid] = tid;
}

650
Samples/3_CUDA_Features/simpleCudaGraphs/simpleCudaGraphs.cu
View File

@@ -32,382 +32,376 @@
namespace cg = cooperative_groups;
#define THREADS_PER_BLOCK 512
#define THREADS_PER_BLOCK 512
#define GRAPH_LAUNCH_ITERATIONS 3
typedef struct callBackData {
const char *fn_name;
double *data;
typedef struct callBackData
{
const char *fn_name;
double *data;
} callBackData_t;
__global__ void reduce(float *inputVec, double *outputVec, size_t inputSize,
size_t outputSize) {
__shared__ double tmp[THREADS_PER_BLOCK];
__global__ void reduce(float *inputVec, double *outputVec, size_t inputSize, size_t outputSize)
{
__shared__ double tmp[THREADS_PER_BLOCK];
cg::thread_block cta = cg::this_thread_block();
size_t globaltid = blockIdx.x * blockDim.x + threadIdx.x;
cg::thread_block cta = cg::this_thread_block();
size_t globaltid = blockIdx.x * blockDim.x + threadIdx.x;
double temp_sum = 0.0;
for (int i = globaltid; i < inputSize; i += gridDim.x * blockDim.x) {
temp_sum += (double)inputVec[i];
}
tmp[cta.thread_rank()] = temp_sum;
cg::sync(cta);
cg::thread_block_tile<32> tile32 = cg::tiled_partition<32>(cta);
double beta = temp_sum;
double temp;
for (int i = tile32.size() / 2; i > 0; i >>= 1) {
if (tile32.thread_rank() < i) {
temp = tmp[cta.thread_rank() + i];
beta += temp;
tmp[cta.thread_rank()] = beta;
double temp_sum = 0.0;
for (int i = globaltid; i < inputSize; i += gridDim.x * blockDim.x) {
temp_sum += (double)inputVec[i];
}
cg::sync(tile32);
}
cg::sync(cta);
tmp[cta.thread_rank()] = temp_sum;
if (cta.thread_rank() == 0 && blockIdx.x < outputSize) {
beta = 0.0;
for (int i = 0; i < cta.size(); i += tile32.size()) {
beta += tmp[i];
cg::sync(cta);
cg::thread_block_tile<32> tile32 = cg::tiled_partition<32>(cta);
double beta = temp_sum;
double temp;
for (int i = tile32.size() / 2; i > 0; i >>= 1) {
if (tile32.thread_rank() < i) {
temp = tmp[cta.thread_rank() + i];
beta += temp;
tmp[cta.thread_rank()] = beta;
}
cg::sync(tile32);
}
outputVec[blockIdx.x] = beta;
}
}
cg::sync(cta);
__global__ void reduceFinal(double *inputVec, double *result,
size_t inputSize) {
__shared__ double tmp[THREADS_PER_BLOCK];
cg::thread_block cta = cg::this_thread_block();
size_t globaltid = blockIdx.x * blockDim.x + threadIdx.x;
double temp_sum = 0.0;
for (int i = globaltid; i < inputSize; i += gridDim.x * blockDim.x) {
temp_sum += (double)inputVec[i];
}
tmp[cta.thread_rank()] = temp_sum;
cg::sync(cta);
cg::thread_block_tile<32> tile32 = cg::tiled_partition<32>(cta);
// do reduction in shared mem
if ((blockDim.x >= 512) && (cta.thread_rank() < 256)) {
tmp[cta.thread_rank()] = temp_sum = temp_sum + tmp[cta.thread_rank() + 256];
}
cg::sync(cta);
if ((blockDim.x >= 256) && (cta.thread_rank() < 128)) {
tmp[cta.thread_rank()] = temp_sum = temp_sum + tmp[cta.thread_rank() + 128];
}
cg::sync(cta);
if ((blockDim.x >= 128) && (cta.thread_rank() < 64)) {
tmp[cta.thread_rank()] = temp_sum = temp_sum + tmp[cta.thread_rank() + 64];
}
cg::sync(cta);
if (cta.thread_rank() < 32) {
// Fetch final intermediate sum from 2nd warp
if (blockDim.x >= 64) temp_sum += tmp[cta.thread_rank() + 32];
// Reduce final warp using shuffle
for (int offset = tile32.size() / 2; offset > 0; offset /= 2) {
temp_sum += tile32.shfl_down(temp_sum, offset);
if (cta.thread_rank() == 0 && blockIdx.x < outputSize) {
beta = 0.0;
for (int i = 0; i < cta.size(); i += tile32.size()) {
beta += tmp[i];
}
outputVec[blockIdx.x] = beta;
}
}
// write result for this block to global mem
if (cta.thread_rank() == 0) result[0] = temp_sum;
}
void init_input(float *a, size_t size) {
for (size_t i = 0; i < size; i++) a[i] = (rand() & 0xFF) / (float)RAND_MAX;
__global__ void reduceFinal(double *inputVec, double *result, size_t inputSize)
{
__shared__ double tmp[THREADS_PER_BLOCK];
cg::thread_block cta = cg::this_thread_block();
size_t globaltid = blockIdx.x * blockDim.x + threadIdx.x;
double temp_sum = 0.0;
for (int i = globaltid; i < inputSize; i += gridDim.x * blockDim.x) {
temp_sum += (double)inputVec[i];
}
tmp[cta.thread_rank()] = temp_sum;
cg::sync(cta);
cg::thread_block_tile<32> tile32 = cg::tiled_partition<32>(cta);
// do reduction in shared mem
if ((blockDim.x >= 512) && (cta.thread_rank() < 256)) {
tmp[cta.thread_rank()] = temp_sum = temp_sum + tmp[cta.thread_rank() + 256];
}
cg::sync(cta);
if ((blockDim.x >= 256) && (cta.thread_rank() < 128)) {
tmp[cta.thread_rank()] = temp_sum = temp_sum + tmp[cta.thread_rank() + 128];
}
cg::sync(cta);
if ((blockDim.x >= 128) && (cta.thread_rank() < 64)) {
tmp[cta.thread_rank()] = temp_sum = temp_sum + tmp[cta.thread_rank() + 64];
}
cg::sync(cta);
if (cta.thread_rank() < 32) {
// Fetch final intermediate sum from 2nd warp
if (blockDim.x >= 64)
temp_sum += tmp[cta.thread_rank() + 32];
// Reduce final warp using shuffle
for (int offset = tile32.size() / 2; offset > 0; offset /= 2) {
temp_sum += tile32.shfl_down(temp_sum, offset);
}
}
// write result for this block to global mem
if (cta.thread_rank() == 0)
result[0] = temp_sum;
}
void CUDART_CB myHostNodeCallback(void *data) {
// Check status of GPU after stream operations are done
callBackData_t *tmp = (callBackData_t *)(data);
// checkCudaErrors(tmp->status);
double *result = (double *)(tmp->data);
char *function = (char *)(tmp->fn_name);
printf("[%s] Host callback final reduced sum = %lf\n", function, *result);
*result = 0.0; // reset the result
void init_input(float *a, size_t size)
{
for (size_t i = 0; i < size; i++)
a[i] = (rand() & 0xFF) / (float)RAND_MAX;
}
void cudaGraphsManual(float *inputVec_h, float *inputVec_d, double *outputVec_d,
double *result_d, size_t inputSize, size_t numOfBlocks) {
cudaStream_t streamForGraph;
cudaGraph_t graph;
std::vector<cudaGraphNode_t> nodeDependencies;
cudaGraphNode_t memcpyNode, kernelNode, memsetNode;
double result_h = 0.0;
void CUDART_CB myHostNodeCallback(void *data)
{
// Check status of GPU after stream operations are done
callBackData_t *tmp = (callBackData_t *)(data);
// checkCudaErrors(tmp->status);
checkCudaErrors(cudaStreamCreate(&streamForGraph));
cudaKernelNodeParams kernelNodeParams = {0};
cudaMemcpy3DParms memcpyParams = {0};
cudaMemsetParams memsetParams = {0};
memcpyParams.srcArray = NULL;
memcpyParams.srcPos = make_cudaPos(0, 0, 0);
memcpyParams.srcPtr =
make_cudaPitchedPtr(inputVec_h, sizeof(float) * inputSize, inputSize, 1);
memcpyParams.dstArray = NULL;
memcpyParams.dstPos = make_cudaPos(0, 0, 0);
memcpyParams.dstPtr =
make_cudaPitchedPtr(inputVec_d, sizeof(float) * inputSize, inputSize, 1);
memcpyParams.extent = make_cudaExtent(sizeof(float) * inputSize, 1, 1);
memcpyParams.kind = cudaMemcpyHostToDevice;
memsetParams.dst = (void *)outputVec_d;
memsetParams.value = 0;
memsetParams.pitch = 0;
memsetParams.elementSize = sizeof(float); // elementSize can be max 4 bytes
memsetParams.width = numOfBlocks * 2;
memsetParams.height = 1;
checkCudaErrors(cudaGraphCreate(&graph, 0));
checkCudaErrors(
cudaGraphAddMemcpyNode(&memcpyNode, graph, NULL, 0, &memcpyParams));
checkCudaErrors(
cudaGraphAddMemsetNode(&memsetNode, graph, NULL, 0, &memsetParams));
nodeDependencies.push_back(memsetNode);
nodeDependencies.push_back(memcpyNode);
void *kernelArgs[4] = {(void *)&inputVec_d, (void *)&outputVec_d, &inputSize,
&numOfBlocks};
kernelNodeParams.func = (void *)reduce;
kernelNodeParams.gridDim = dim3(numOfBlocks, 1, 1);
kernelNodeParams.blockDim = dim3(THREADS_PER_BLOCK, 1, 1);
kernelNodeParams.sharedMemBytes = 0;
kernelNodeParams.kernelParams = (void **)kernelArgs;
kernelNodeParams.extra = NULL;
checkCudaErrors(
cudaGraphAddKernelNode(&kernelNode, graph, nodeDependencies.data(),
nodeDependencies.size(), &kernelNodeParams));
nodeDependencies.clear();
nodeDependencies.push_back(kernelNode);
memset(&memsetParams, 0, sizeof(memsetParams));
memsetParams.dst = result_d;
memsetParams.value = 0;
memsetParams.elementSize = sizeof(float);
memsetParams.width = 2;
memsetParams.height = 1;
checkCudaErrors(
cudaGraphAddMemsetNode(&memsetNode, graph, NULL, 0, &memsetParams));
nodeDependencies.push_back(memsetNode);
memset(&kernelNodeParams, 0, sizeof(kernelNodeParams));
kernelNodeParams.func = (void *)reduceFinal;
kernelNodeParams.gridDim = dim3(1, 1, 1);
kernelNodeParams.blockDim = dim3(THREADS_PER_BLOCK, 1, 1);
kernelNodeParams.sharedMemBytes = 0;
void *kernelArgs2[3] = {(void *)&outputVec_d, (void *)&result_d,
&numOfBlocks};
kernelNodeParams.kernelParams = kernelArgs2;
kernelNodeParams.extra = NULL;
checkCudaErrors(
cudaGraphAddKernelNode(&kernelNode, graph, nodeDependencies.data(),
nodeDependencies.size(), &kernelNodeParams));
nodeDependencies.clear();
nodeDependencies.push_back(kernelNode);
memset(&memcpyParams, 0, sizeof(memcpyParams));
memcpyParams.srcArray = NULL;
memcpyParams.srcPos = make_cudaPos(0, 0, 0);
memcpyParams.srcPtr = make_cudaPitchedPtr(result_d, sizeof(double), 1, 1);
memcpyParams.dstArray = NULL;
memcpyParams.dstPos = make_cudaPos(0, 0, 0);
memcpyParams.dstPtr = make_cudaPitchedPtr(&result_h, sizeof(double), 1, 1);
memcpyParams.extent = make_cudaExtent(sizeof(double), 1, 1);
memcpyParams.kind = cudaMemcpyDeviceToHost;
checkCudaErrors(
cudaGraphAddMemcpyNode(&memcpyNode, graph, nodeDependencies.data(),
nodeDependencies.size(), &memcpyParams));
nodeDependencies.clear();
nodeDependencies.push_back(memcpyNode);
cudaGraphNode_t hostNode;
cudaHostNodeParams hostParams = {0};
hostParams.fn = myHostNodeCallback;
callBackData_t hostFnData;
hostFnData.data = &result_h;
hostFnData.fn_name = "cudaGraphsManual";
hostParams.userData = &hostFnData;
checkCudaErrors(cudaGraphAddHostNode(&hostNode, graph,
nodeDependencies.data(),
nodeDependencies.size(), &hostParams));
cudaGraphNode_t *nodes = NULL;
size_t numNodes = 0;
checkCudaErrors(cudaGraphGetNodes(graph, nodes, &numNodes));
printf("\nNum of nodes in the graph created manually = %zu\n", numNodes);
cudaGraphExec_t graphExec;
checkCudaErrors(cudaGraphInstantiate(&graphExec, graph, NULL, NULL, 0));
cudaGraph_t clonedGraph;
cudaGraphExec_t clonedGraphExec;
checkCudaErrors(cudaGraphClone(&clonedGraph, graph));
checkCudaErrors(
cudaGraphInstantiate(&clonedGraphExec, clonedGraph, NULL, NULL, 0));
for (int i = 0; i < GRAPH_LAUNCH_ITERATIONS; i++) {
checkCudaErrors(cudaGraphLaunch(graphExec, streamForGraph));
}
checkCudaErrors(cudaStreamSynchronize(streamForGraph));
printf("Cloned Graph Output.. \n");
for (int i = 0; i < GRAPH_LAUNCH_ITERATIONS; i++) {
checkCudaErrors(cudaGraphLaunch(clonedGraphExec, streamForGraph));
}
checkCudaErrors(cudaStreamSynchronize(streamForGraph));
checkCudaErrors(cudaGraphExecDestroy(graphExec));
checkCudaErrors(cudaGraphExecDestroy(clonedGraphExec));
checkCudaErrors(cudaGraphDestroy(graph));
checkCudaErrors(cudaGraphDestroy(clonedGraph));
checkCudaErrors(cudaStreamDestroy(streamForGraph));
double *result = (double *)(tmp->data);
char *function = (char *)(tmp->fn_name);
printf("[%s] Host callback final reduced sum = %lf\n", function, *result);
*result = 0.0; // reset the result
}
void cudaGraphsUsingStreamCapture(float *inputVec_h, float *inputVec_d,
double *outputVec_d, double *result_d,
size_t inputSize, size_t numOfBlocks) {
cudaStream_t stream1, stream2, stream3, streamForGraph;
cudaEvent_t forkStreamEvent, memsetEvent1, memsetEvent2;
cudaGraph_t graph;
double result_h = 0.0;
void cudaGraphsManual(float *inputVec_h,
float *inputVec_d,
double *outputVec_d,
double *result_d,
size_t inputSize,
size_t numOfBlocks)
{
cudaStream_t streamForGraph;
cudaGraph_t graph;
std::vector<cudaGraphNode_t> nodeDependencies;
cudaGraphNode_t memcpyNode, kernelNode, memsetNode;
double result_h = 0.0;
checkCudaErrors(cudaStreamCreate(&stream1));
checkCudaErrors(cudaStreamCreate(&stream2));
checkCudaErrors(cudaStreamCreate(&stream3));
checkCudaErrors(cudaStreamCreate(&streamForGraph));
checkCudaErrors(cudaStreamCreate(&streamForGraph));
checkCudaErrors(cudaEventCreate(&forkStreamEvent));
checkCudaErrors(cudaEventCreate(&memsetEvent1));
checkCudaErrors(cudaEventCreate(&memsetEvent2));
cudaKernelNodeParams kernelNodeParams = {0};
cudaMemcpy3DParms memcpyParams = {0};
cudaMemsetParams memsetParams = {0};
checkCudaErrors(cudaStreamBeginCapture(stream1, cudaStreamCaptureModeGlobal));
memcpyParams.srcArray = NULL;
memcpyParams.srcPos = make_cudaPos(0, 0, 0);
memcpyParams.srcPtr = make_cudaPitchedPtr(inputVec_h, sizeof(float) * inputSize, inputSize, 1);
memcpyParams.dstArray = NULL;
memcpyParams.dstPos = make_cudaPos(0, 0, 0);
memcpyParams.dstPtr = make_cudaPitchedPtr(inputVec_d, sizeof(float) * inputSize, inputSize, 1);
memcpyParams.extent = make_cudaExtent(sizeof(float) * inputSize, 1, 1);
memcpyParams.kind = cudaMemcpyHostToDevice;
checkCudaErrors(cudaEventRecord(forkStreamEvent, stream1));
checkCudaErrors(cudaStreamWaitEvent(stream2, forkStreamEvent, 0));
checkCudaErrors(cudaStreamWaitEvent(stream3, forkStreamEvent, 0));
memsetParams.dst = (void *)outputVec_d;
memsetParams.value = 0;
memsetParams.pitch = 0;
memsetParams.elementSize = sizeof(float); // elementSize can be max 4 bytes
memsetParams.width = numOfBlocks * 2;
memsetParams.height = 1;
checkCudaErrors(cudaMemcpyAsync(inputVec_d, inputVec_h,
sizeof(float) * inputSize, cudaMemcpyDefault,
stream1));
checkCudaErrors(cudaGraphCreate(&graph, 0));
checkCudaErrors(cudaGraphAddMemcpyNode(&memcpyNode, graph, NULL, 0, &memcpyParams));
checkCudaErrors(cudaGraphAddMemsetNode(&memsetNode, graph, NULL, 0, &memsetParams));
checkCudaErrors(
cudaMemsetAsync(outputVec_d, 0, sizeof(double) * numOfBlocks, stream2));
nodeDependencies.push_back(memsetNode);
nodeDependencies.push_back(memcpyNode);
checkCudaErrors(cudaEventRecord(memsetEvent1, stream2));
void *kernelArgs[4] = {(void *)&inputVec_d, (void *)&outputVec_d, &inputSize, &numOfBlocks};
checkCudaErrors(cudaMemsetAsync(result_d, 0, sizeof(double), stream3));
checkCudaErrors(cudaEventRecord(memsetEvent2, stream3));
kernelNodeParams.func = (void *)reduce;
kernelNodeParams.gridDim = dim3(numOfBlocks, 1, 1);
kernelNodeParams.blockDim = dim3(THREADS_PER_BLOCK, 1, 1);
kernelNodeParams.sharedMemBytes = 0;
kernelNodeParams.kernelParams = (void **)kernelArgs;
kernelNodeParams.extra = NULL;
checkCudaErrors(cudaStreamWaitEvent(stream1, memsetEvent1, 0));
checkCudaErrors(cudaGraphAddKernelNode(
&kernelNode, graph, nodeDependencies.data(), nodeDependencies.size(), &kernelNodeParams));
reduce<<<numOfBlocks, THREADS_PER_BLOCK, 0, stream1>>>(
inputVec_d, outputVec_d, inputSize, numOfBlocks);
nodeDependencies.clear();
nodeDependencies.push_back(kernelNode);
checkCudaErrors(cudaStreamWaitEvent(stream1, memsetEvent2, 0));
memset(&memsetParams, 0, sizeof(memsetParams));
memsetParams.dst = result_d;
memsetParams.value = 0;
memsetParams.elementSize = sizeof(float);
memsetParams.width = 2;
memsetParams.height = 1;
checkCudaErrors(cudaGraphAddMemsetNode(&memsetNode, graph, NULL, 0, &memsetParams));
reduceFinal<<<1, THREADS_PER_BLOCK, 0, stream1>>>(outputVec_d, result_d,
numOfBlocks);
checkCudaErrors(cudaMemcpyAsync(&result_h, result_d, sizeof(double),
cudaMemcpyDefault, stream1));
nodeDependencies.push_back(memsetNode);
callBackData_t hostFnData = {0};
hostFnData.data = &result_h;
hostFnData.fn_name = "cudaGraphsUsingStreamCapture";
cudaHostFn_t fn = myHostNodeCallback;
checkCudaErrors(cudaLaunchHostFunc(stream1, fn, &hostFnData));
checkCudaErrors(cudaStreamEndCapture(stream1, &graph));
memset(&kernelNodeParams, 0, sizeof(kernelNodeParams));
kernelNodeParams.func = (void *)reduceFinal;
kernelNodeParams.gridDim = dim3(1, 1, 1);
kernelNodeParams.blockDim = dim3(THREADS_PER_BLOCK, 1, 1);
kernelNodeParams.sharedMemBytes = 0;
void *kernelArgs2[3] = {(void *)&outputVec_d, (void *)&result_d, &numOfBlocks};
kernelNodeParams.kernelParams = kernelArgs2;
kernelNodeParams.extra = NULL;
cudaGraphNode_t *nodes = NULL;
size_t numNodes = 0;
checkCudaErrors(cudaGraphGetNodes(graph, nodes, &numNodes));
printf("\nNum of nodes in the graph created using stream capture API = %zu\n",
numNodes);
checkCudaErrors(cudaGraphAddKernelNode(
&kernelNode, graph, nodeDependencies.data(), nodeDependencies.size(), &kernelNodeParams));
nodeDependencies.clear();
nodeDependencies.push_back(kernelNode);
cudaGraphExec_t graphExec;
checkCudaErrors(cudaGraphInstantiate(&graphExec, graph, NULL, NULL, 0));
memset(&memcpyParams, 0, sizeof(memcpyParams));
cudaGraph_t clonedGraph;
cudaGraphExec_t clonedGraphExec;
checkCudaErrors(cudaGraphClone(&clonedGraph, graph));
checkCudaErrors(
cudaGraphInstantiate(&clonedGraphExec, clonedGraph, NULL, NULL, 0));
memcpyParams.srcArray = NULL;
memcpyParams.srcPos = make_cudaPos(0, 0, 0);
memcpyParams.srcPtr = make_cudaPitchedPtr(result_d, sizeof(double), 1, 1);
memcpyParams.dstArray = NULL;
memcpyParams.dstPos = make_cudaPos(0, 0, 0);
memcpyParams.dstPtr = make_cudaPitchedPtr(&result_h, sizeof(double), 1, 1);
memcpyParams.extent = make_cudaExtent(sizeof(double), 1, 1);
memcpyParams.kind = cudaMemcpyDeviceToHost;
checkCudaErrors(
cudaGraphAddMemcpyNode(&memcpyNode, graph, nodeDependencies.data(), nodeDependencies.size(), &memcpyParams));
nodeDependencies.clear();
nodeDependencies.push_back(memcpyNode);
for (int i = 0; i < GRAPH_LAUNCH_ITERATIONS; i++) {
checkCudaErrors(cudaGraphLaunch(graphExec, streamForGraph));
}
cudaGraphNode_t hostNode;
cudaHostNodeParams hostParams = {0};
hostParams.fn = myHostNodeCallback;
callBackData_t hostFnData;
hostFnData.data = &result_h;
hostFnData.fn_name = "cudaGraphsManual";
hostParams.userData = &hostFnData;
checkCudaErrors(cudaStreamSynchronize(streamForGraph));
checkCudaErrors(
cudaGraphAddHostNode(&hostNode, graph, nodeDependencies.data(), nodeDependencies.size(), &hostParams));
printf("Cloned Graph Output.. \n");
for (int i = 0; i < GRAPH_LAUNCH_ITERATIONS; i++) {
checkCudaErrors(cudaGraphLaunch(clonedGraphExec, streamForGraph));
}
cudaGraphNode_t *nodes = NULL;
size_t numNodes = 0;
checkCudaErrors(cudaGraphGetNodes(graph, nodes, &numNodes));
printf("\nNum of nodes in the graph created manually = %zu\n", numNodes);
checkCudaErrors(cudaStreamSynchronize(streamForGraph));
cudaGraphExec_t graphExec;
checkCudaErrors(cudaGraphInstantiate(&graphExec, graph, NULL, NULL, 0));
checkCudaErrors(cudaGraphExecDestroy(graphExec));
checkCudaErrors(cudaGraphExecDestroy(clonedGraphExec));
checkCudaErrors(cudaGraphDestroy(graph));
checkCudaErrors(cudaGraphDestroy(clonedGraph));
checkCudaErrors(cudaStreamDestroy(stream1));
checkCudaErrors(cudaStreamDestroy(stream2));
checkCudaErrors(cudaStreamDestroy(streamForGraph));
cudaGraph_t clonedGraph;
cudaGraphExec_t clonedGraphExec;
checkCudaErrors(cudaGraphClone(&clonedGraph, graph));
checkCudaErrors(cudaGraphInstantiate(&clonedGraphExec, clonedGraph, NULL, NULL, 0));
for (int i = 0; i < GRAPH_LAUNCH_ITERATIONS; i++) {
checkCudaErrors(cudaGraphLaunch(graphExec, streamForGraph));
}
checkCudaErrors(cudaStreamSynchronize(streamForGraph));
printf("Cloned Graph Output.. \n");
for (int i = 0; i < GRAPH_LAUNCH_ITERATIONS; i++) {
checkCudaErrors(cudaGraphLaunch(clonedGraphExec, streamForGraph));
}
checkCudaErrors(cudaStreamSynchronize(streamForGraph));
checkCudaErrors(cudaGraphExecDestroy(graphExec));
checkCudaErrors(cudaGraphExecDestroy(clonedGraphExec));
checkCudaErrors(cudaGraphDestroy(graph));
checkCudaErrors(cudaGraphDestroy(clonedGraph));
checkCudaErrors(cudaStreamDestroy(streamForGraph));
}
int main(int argc, char **argv) {
size_t size = 1 << 24; // number of elements to reduce
size_t maxBlocks = 512;
void cudaGraphsUsingStreamCapture(float *inputVec_h,
float *inputVec_d,
double *outputVec_d,
double *result_d,
size_t inputSize,
size_t numOfBlocks)
{
cudaStream_t stream1, stream2, stream3, streamForGraph;
cudaEvent_t forkStreamEvent, memsetEvent1, memsetEvent2;
cudaGraph_t graph;
double result_h = 0.0;
// This will pick the best possible CUDA capable device
int devID = findCudaDevice(argc, (const char **)argv);
checkCudaErrors(cudaStreamCreate(&stream1));
checkCudaErrors(cudaStreamCreate(&stream2));
checkCudaErrors(cudaStreamCreate(&stream3));
checkCudaErrors(cudaStreamCreate(&streamForGraph));
printf("%zu elements\n", size);
printf("threads per block = %d\n", THREADS_PER_BLOCK);
printf("Graph Launch iterations = %d\n", GRAPH_LAUNCH_ITERATIONS);
checkCudaErrors(cudaEventCreate(&forkStreamEvent));
checkCudaErrors(cudaEventCreate(&memsetEvent1));
checkCudaErrors(cudaEventCreate(&memsetEvent2));
float *inputVec_d = NULL, *inputVec_h = NULL;
double *outputVec_d = NULL, *result_d;
checkCudaErrors(cudaStreamBeginCapture(stream1, cudaStreamCaptureModeGlobal));
checkCudaErrors(cudaMallocHost(&inputVec_h, sizeof(float) * size));
checkCudaErrors(cudaMalloc(&inputVec_d, sizeof(float) * size));
checkCudaErrors(cudaMalloc(&outputVec_d, sizeof(double) * maxBlocks));
checkCudaErrors(cudaMalloc(&result_d, sizeof(double)));
checkCudaErrors(cudaEventRecord(forkStreamEvent, stream1));
checkCudaErrors(cudaStreamWaitEvent(stream2, forkStreamEvent, 0));
checkCudaErrors(cudaStreamWaitEvent(stream3, forkStreamEvent, 0));
init_input(inputVec_h, size);
checkCudaErrors(cudaMemcpyAsync(inputVec_d, inputVec_h, sizeof(float) * inputSize, cudaMemcpyDefault, stream1));
cudaGraphsManual(inputVec_h, inputVec_d, outputVec_d, result_d, size,
maxBlocks);
cudaGraphsUsingStreamCapture(inputVec_h, inputVec_d, outputVec_d, result_d,
size, maxBlocks);
checkCudaErrors(cudaMemsetAsync(outputVec_d, 0, sizeof(double) * numOfBlocks, stream2));
checkCudaErrors(cudaFree(inputVec_d));
checkCudaErrors(cudaFree(outputVec_d));
checkCudaErrors(cudaFree(result_d));
checkCudaErrors(cudaFreeHost(inputVec_h));
return EXIT_SUCCESS;
checkCudaErrors(cudaEventRecord(memsetEvent1, stream2));
checkCudaErrors(cudaMemsetAsync(result_d, 0, sizeof(double), stream3));
checkCudaErrors(cudaEventRecord(memsetEvent2, stream3));
checkCudaErrors(cudaStreamWaitEvent(stream1, memsetEvent1, 0));
reduce<<<numOfBlocks, THREADS_PER_BLOCK, 0, stream1>>>(inputVec_d, outputVec_d, inputSize, numOfBlocks);
checkCudaErrors(cudaStreamWaitEvent(stream1, memsetEvent2, 0));
reduceFinal<<<1, THREADS_PER_BLOCK, 0, stream1>>>(outputVec_d, result_d, numOfBlocks);
checkCudaErrors(cudaMemcpyAsync(&result_h, result_d, sizeof(double), cudaMemcpyDefault, stream1));
callBackData_t hostFnData = {0};
hostFnData.data = &result_h;
hostFnData.fn_name = "cudaGraphsUsingStreamCapture";
cudaHostFn_t fn = myHostNodeCallback;
checkCudaErrors(cudaLaunchHostFunc(stream1, fn, &hostFnData));
checkCudaErrors(cudaStreamEndCapture(stream1, &graph));
cudaGraphNode_t *nodes = NULL;
size_t numNodes = 0;
checkCudaErrors(cudaGraphGetNodes(graph, nodes, &numNodes));
printf("\nNum of nodes in the graph created using stream capture API = %zu\n", numNodes);
cudaGraphExec_t graphExec;
checkCudaErrors(cudaGraphInstantiate(&graphExec, graph, NULL, NULL, 0));
cudaGraph_t clonedGraph;
cudaGraphExec_t clonedGraphExec;
checkCudaErrors(cudaGraphClone(&clonedGraph, graph));
checkCudaErrors(cudaGraphInstantiate(&clonedGraphExec, clonedGraph, NULL, NULL, 0));
for (int i = 0; i < GRAPH_LAUNCH_ITERATIONS; i++) {
checkCudaErrors(cudaGraphLaunch(graphExec, streamForGraph));
}
checkCudaErrors(cudaStreamSynchronize(streamForGraph));
printf("Cloned Graph Output.. \n");
for (int i = 0; i < GRAPH_LAUNCH_ITERATIONS; i++) {
checkCudaErrors(cudaGraphLaunch(clonedGraphExec, streamForGraph));
}
checkCudaErrors(cudaStreamSynchronize(streamForGraph));
checkCudaErrors(cudaGraphExecDestroy(graphExec));
checkCudaErrors(cudaGraphExecDestroy(clonedGraphExec));
checkCudaErrors(cudaGraphDestroy(graph));
checkCudaErrors(cudaGraphDestroy(clonedGraph));
checkCudaErrors(cudaStreamDestroy(stream1));
checkCudaErrors(cudaStreamDestroy(stream2));
checkCudaErrors(cudaStreamDestroy(streamForGraph));
}
int main(int argc, char **argv)
{
size_t size = 1 << 24; // number of elements to reduce
size_t maxBlocks = 512;
// This will pick the best possible CUDA capable device
int devID = findCudaDevice(argc, (const char **)argv);
printf("%zu elements\n", size);
printf("threads per block = %d\n", THREADS_PER_BLOCK);
printf("Graph Launch iterations = %d\n", GRAPH_LAUNCH_ITERATIONS);
float *inputVec_d = NULL, *inputVec_h = NULL;
double *outputVec_d = NULL, *result_d;
checkCudaErrors(cudaMallocHost(&inputVec_h, sizeof(float) * size));
checkCudaErrors(cudaMalloc(&inputVec_d, sizeof(float) * size));
checkCudaErrors(cudaMalloc(&outputVec_d, sizeof(double) * maxBlocks));
checkCudaErrors(cudaMalloc(&result_d, sizeof(double)));
init_input(inputVec_h, size);
cudaGraphsManual(inputVec_h, inputVec_d, outputVec_d, result_d, size, maxBlocks);
cudaGraphsUsingStreamCapture(inputVec_h, inputVec_d, outputVec_d, result_d, size, maxBlocks);
checkCudaErrors(cudaFree(inputVec_d));
checkCudaErrors(cudaFree(outputVec_d));
checkCudaErrors(cudaFree(result_d));
checkCudaErrors(cudaFreeHost(inputVec_h));
return EXIT_SUCCESS;
}

344
Samples/3_CUDA_Features/tf32TensorCoreGemm/tf32TensorCoreGemm.cu
View File

@@ -58,14 +58,14 @@
// but carefully enough to avoid local memory use.
#include <assert.h>
#include <stdio.h>
#include <cuda.h>
#include <mma.h>
#include <cuda/pipeline>
#include <mma.h>
#include <stdio.h>
// helper functions and utilities to work with CUDA
#include <helper_functions.h>
#include <helper_cuda.h>
#include <helper_functions.h>
// Externally configurable parameters.
@@ -106,7 +106,7 @@
// Implementation constants.
#define WARPS_PER_BLOCK 8
#define WARPS_PER_BLOCK 8
#define THREADS_PER_BLOCK (WARP_SIZE * WARPS_PER_BLOCK)
#if SHARED_MEMORY_LIMIT_64K
@@ -124,10 +124,10 @@
#define CHUNK_K 8
#endif
#define CHUNK_LINE_BYTES (CHUNK_K * K * sizeof(float))
#define WARP_COPY_BYTES (WARP_SIZE * sizeof(int4))
#define CHUNK_LINE_BYTES (CHUNK_K * K * sizeof(float))
#define WARP_COPY_BYTES (WARP_SIZE * sizeof(int4))
#define CHUNK_COPY_LINES_PER_WARP (WARP_COPY_BYTES / CHUNK_LINE_BYTES)
#define CHUNK_COPY_LINE_LANES (WARP_SIZE / CHUNK_COPY_LINES_PER_WARP)
#define CHUNK_COPY_LINE_LANES (WARP_SIZE / CHUNK_COPY_LINES_PER_WARP)
#define BLOCK_ROW_WARPS 2
#define BLOCK_COL_WARPS 4
@@ -157,25 +157,24 @@
// we must keep each row and column 256-bit aligned, as required by nvcuda::wmma::load_matrix_sync.
#define SKEW_FLOAT 8
#define checkKernelErrors(expr) do { \
expr; \
\
cudaError_t __err = cudaGetLastError(); \
if (__err != cudaSuccess) { \
printf("Line %d: '%s' failed: %s\n", __LINE__, # expr, cudaGetErrorString(__err)); \
abort(); \
} \
} while(0)
#define checkKernelErrors(expr) \
do { \
expr; \
\
cudaError_t __err = cudaGetLastError(); \
if (__err != cudaSuccess) { \
printf("Line %d: '%s' failed: %s\n", __LINE__, #expr, cudaGetErrorString(__err)); \
abort(); \
} \
} while (0)
enum kernels
{
tf32mma_shmem_gemm_async_copy = 0, // tf32 MMA shmem using kernel with async_copy
tf32mma_shmem_gemm = 1, // tf32 MMA shmem using kernel normal copy (without async_copy).
simple_tf32mma_gemm = 2 // tf32 MMA non-shmem using simple kernel.
enum kernels {
tf32mma_shmem_gemm_async_copy = 0, // tf32 MMA shmem using kernel with async_copy
tf32mma_shmem_gemm = 1, // tf32 MMA shmem using kernel normal copy (without async_copy).
simple_tf32mma_gemm = 2 // tf32 MMA non-shmem using simple kernel.
};
const char* kernelNames[] = {"compute_tf32gemm_async_copy", "compute_tf32gemm",
"simple_wmma_tf32gemm"};
const char *kernelNames[] = {"compute_tf32gemm_async_copy", "compute_tf32gemm", "simple_wmma_tf32gemm"};
using namespace nvcuda;
@@ -183,18 +182,18 @@ __host__ void init_host_matrices(float *a, float *b, float *c)
{
for (int i = 0; i < M_GLOBAL; i++) {
for (int j = 0; j < K_GLOBAL; j++) {
a[i*K_GLOBAL+j] = (float)(rand() % 3);
a[i * K_GLOBAL + j] = (float)(rand() % 3);
}
}
for (int i = 0; i < N_GLOBAL; i++) {
for (int j = 0; j < K_GLOBAL; j++) {
b[i*K_GLOBAL+j] = (float)(rand() % 3);
b[i * K_GLOBAL + j] = (float)(rand() % 3);
}
}
for (int t = 0; t < M_GLOBAL * N_GLOBAL; t++) {
c[t] = (float)(rand() % 3);
c[t] = (float)(rand() % 3);
}
}
@@ -211,10 +210,11 @@ __global__ void compute_tf32gemm(const float *A, const float *B, const float *C,
const size_t shmem_idx_b_off = BLOCK_COL_TILES * M;
// This pointer is used to access the C and D matrix tiles this warp computes.
float *shmem_warp_tile_ptr = (float*)&shmem[0][0] + (warpId / BLOCK_ROW_WARPS) * SHMEM_STRIDE * N * BLOCK_ROW_WARPS + (warpId % BLOCK_ROW_WARPS) * SHMEM_OFFSET;
float *shmem_warp_tile_ptr = (float *)&shmem[0][0] + (warpId / BLOCK_ROW_WARPS) * SHMEM_STRIDE * N * BLOCK_ROW_WARPS
+ (warpId % BLOCK_ROW_WARPS) * SHMEM_OFFSET;
// This pointer is used to stream the C and D matrices block-wide tile to and from shared memory.
float *shmem_warp_stream_ptr = (float*)&shmem[0][0] + warpId * SHMEM_STRIDE * N;
float *shmem_warp_stream_ptr = (float *)&shmem[0][0] + warpId * SHMEM_STRIDE * N;
// Adjust the beta scaler, as it'll be multiplied by alpha at the end of
// each tile computation. Technically this is not generally correct (may result
@@ -224,7 +224,7 @@ __global__ void compute_tf32gemm(const float *A, const float *B, const float *C,
// Each CTA slides along the 128 x 128 tiles from the top left corner of the matrix to the
// right and down, and selects the next tile to compute. Once there's no such tile,
// all warps in this CTA exit.
for(unsigned int block_pos = blockIdx.x;; block_pos += gridDim.x) {
for (unsigned int block_pos = blockIdx.x;; block_pos += gridDim.x) {
const unsigned int block_tile_i = ((block_pos * BLOCK_ROW_TILES) / N_TILES) * (BLOCK_COL_TILES);
const unsigned int block_tile_j = (block_pos * BLOCK_COL_TILES) % N_TILES;
@@ -234,14 +234,14 @@ __global__ void compute_tf32gemm(const float *A, const float *B, const float *C,
}
// This warp's pointer to the C matrix data to copy memory from to shared memory.
const size_t gmem_idx = (block_tile_i + warpId) * M * GLOBAL_MEM_STRIDE + block_tile_j * N;
const size_t gmem_idx = (block_tile_i + warpId) * M * GLOBAL_MEM_STRIDE + block_tile_j * N;
const float *src_gmem_warp_stream_ptr = &C[gmem_idx];
// Stream multiple C tiles to shared memory.
#pragma unroll
for (int i = 0; i < N; i++) {
*((int4*)(shmem_warp_stream_ptr + SHMEM_STRIDE * i) + laneId) =
*((int4*)(src_gmem_warp_stream_ptr + GLOBAL_MEM_STRIDE * i) + laneId);
*((int4 *)(shmem_warp_stream_ptr + SHMEM_STRIDE * i) + laneId) =
*((int4 *)(src_gmem_warp_stream_ptr + GLOBAL_MEM_STRIDE * i) + laneId);
}
__syncthreads();
@@ -265,7 +265,7 @@ __global__ void compute_tf32gemm(const float *A, const float *B, const float *C,
// Scale the C matrix.
#pragma unroll
for (int i = 0; i < WARP_COL_TILES; i++) {
for (int i = 0; i < WARP_COL_TILES; i++) {
#pragma unroll
for (int j = 0; j < WARP_ROW_TILES; j++) {
#pragma unroll
@@ -277,16 +277,19 @@ __global__ void compute_tf32gemm(const float *A, const float *B, const float *C,
// Select what warp copies what matrix to shared memory.
// Warps 0-3 copy the A matrix, warps 4-7 copy the B matrix.
const float *warp_ptr = (warpId < (WARPS_PER_BLOCK/2)) ? (&A[block_tile_i * M * K_GLOBAL] + M * K_GLOBAL * (warpId % (WARPS_PER_BLOCK/2)) * 2) :
(&B[block_tile_j * N * K_GLOBAL] + N * K_GLOBAL * (warpId % (WARPS_PER_BLOCK/2)) * 2);
const float *warp_ptr =
(warpId < (WARPS_PER_BLOCK / 2))
? (&A[block_tile_i * M * K_GLOBAL] + M * K_GLOBAL * (warpId % (WARPS_PER_BLOCK / 2)) * 2)
: (&B[block_tile_j * N * K_GLOBAL] + N * K_GLOBAL * (warpId % (WARPS_PER_BLOCK / 2)) * 2);
// Go through the global K dimension by a fixed step at a time.
#pragma unroll
for (int tile_k = 0; tile_k < K_TILES; tile_k += CHUNK_K) {
// Copy slices of the A and B matrices to shared memory.
// The first half of the warps in the CTA copy the A matrix, the rest copy the B matrix.
size_t shmem_idx = warpId < (WARPS_PER_BLOCK/2) ? (M * (warpId % (WARPS_PER_BLOCK/2)) * 2) :
(N * (warpId % (WARPS_PER_BLOCK/2)) * 2 + shmem_idx_b_off);
size_t shmem_idx = warpId < (WARPS_PER_BLOCK / 2)
? (M * (warpId % (WARPS_PER_BLOCK / 2)) * 2)
: (N * (warpId % (WARPS_PER_BLOCK / 2)) * 2 + shmem_idx_b_off);
// First half of the warp copies the first row / column of the matrix,
// the second half of the warp copies the next.
@@ -296,9 +299,10 @@ __global__ void compute_tf32gemm(const float *A, const float *B, const float *C,
shmem_idx += laneId / CHUNK_COPY_LINE_LANES;
#pragma unroll
for(int i = 0; i < ((WARP_SIZE/2) / CHUNK_COPY_LINES_PER_WARP) * 2; i++) {
for (int i = 0; i < ((WARP_SIZE / 2) / CHUNK_COPY_LINES_PER_WARP) * 2; i++) {
// Copy 16 bytes at once in each lane.
*((int4*)&shmem[shmem_idx][0] + (laneId % CHUNK_COPY_LINE_LANES)) = *((int4*)lane_ptr + (laneId % CHUNK_COPY_LINE_LANES));
*((int4 *)&shmem[shmem_idx][0] + (laneId % CHUNK_COPY_LINE_LANES)) =
*((int4 *)lane_ptr + (laneId % CHUNK_COPY_LINE_LANES));
// Advance the global memory pointer and the shared memory index.
lane_ptr = lane_ptr + K_GLOBAL * CHUNK_COPY_LINES_PER_WARP;
@@ -315,8 +319,8 @@ __global__ void compute_tf32gemm(const float *A, const float *B, const float *C,
#pragma unroll
for (int i = 0; i < WARP_COL_TILES; i++) {
size_t shmem_idx_a = (warpId/BLOCK_ROW_WARPS) * M * BLOCK_ROW_WARPS + (i * M);
const float *tile_ptr = &shmem[shmem_idx_a][k_step * K];
size_t shmem_idx_a = (warpId / BLOCK_ROW_WARPS) * M * BLOCK_ROW_WARPS + (i * M);
const float *tile_ptr = &shmem[shmem_idx_a][k_step * K];
wmma::load_matrix_sync(a[i], tile_ptr, K * CHUNK_K + SKEW_FLOAT);
#pragma unroll
@@ -328,8 +332,8 @@ __global__ void compute_tf32gemm(const float *A, const float *B, const float *C,
if (i == 0) {
// Load the B matrix fragment once, because it is going to be reused
// against the other A matrix fragments.
size_t shmem_idx_b = shmem_idx_b_off + (WARP_ROW_TILES * N) * (warpId%2) + (j * N);
const float *tile_ptr = &shmem[shmem_idx_b][k_step * K];
size_t shmem_idx_b = shmem_idx_b_off + (WARP_ROW_TILES * N) * (warpId % 2) + (j * N);
const float *tile_ptr = &shmem[shmem_idx_b][k_step * K];
wmma::load_matrix_sync(b[j], tile_ptr, K * CHUNK_K + SKEW_FLOAT);
#pragma unroll
@@ -370,8 +374,8 @@ __global__ void compute_tf32gemm(const float *A, const float *B, const float *C,
#pragma unroll
for (int i = 0; i < N; i++) {
*((int4*)(dst_gmem_warp_stream_ptr + GLOBAL_MEM_STRIDE * i) + laneId) =
*((int4*)(shmem_warp_stream_ptr + SHMEM_STRIDE * i) + laneId);
*((int4 *)(dst_gmem_warp_stream_ptr + GLOBAL_MEM_STRIDE * i) + laneId) =
*((int4 *)(shmem_warp_stream_ptr + SHMEM_STRIDE * i) + laneId);
}
__syncthreads();
@@ -379,7 +383,8 @@ __global__ void compute_tf32gemm(const float *A, const float *B, const float *C,
#endif
}
__global__ void compute_tf32gemm_async_copy(const float *A, const float *B, const float *C, float *D, const float alpha, float beta)
__global__ void
compute_tf32gemm_async_copy(const float *A, const float *B, const float *C, float *D, const float alpha, float beta)
{
#if __CUDA_ARCH__ >= 800
extern __shared__ float shmem[][CHUNK_K * K + SKEW_FLOAT];
@@ -389,10 +394,11 @@ __global__ void compute_tf32gemm_async_copy(const float *A, const float *B, cons
const unsigned int laneId = threadIdx.x % WARP_SIZE;
// This pointer is used to access the C and D matrix tiles this warp computes.
float *shmem_warp_tile_ptr = (float*)&shmem[0][0] + (warpId / BLOCK_ROW_WARPS) * SHMEM_STRIDE * N * BLOCK_ROW_WARPS + (warpId % BLOCK_ROW_WARPS) * SHMEM_OFFSET;
float *shmem_warp_tile_ptr = (float *)&shmem[0][0] + (warpId / BLOCK_ROW_WARPS) * SHMEM_STRIDE * N * BLOCK_ROW_WARPS
+ (warpId % BLOCK_ROW_WARPS) * SHMEM_OFFSET;
// This pointer is used to stream the C and D matrices block-wide tile to and from shared memory.
float *shmem_warp_stream_ptr = (float*)&shmem[0][0] + warpId * SHMEM_STRIDE * N;
float *shmem_warp_stream_ptr = (float *)&shmem[0][0] + warpId * SHMEM_STRIDE * N;
// Offset in shared memory from which the B matrix is stored.
constexpr size_t shmem_idx_b_off = BLOCK_COL_TILES * M;
@@ -402,14 +408,14 @@ __global__ void compute_tf32gemm_async_copy(const float *A, const float *B, cons
// in a loss of precision). Zero still needs to be specially handled though.
beta /= alpha;
cuda::pipeline<cuda::thread_scope_thread> pipe = cuda::make_pipeline();
const auto shape4 = cuda::aligned_size_t<alignof(float4)>(sizeof(float4));
constexpr int loadStride = 2; // load 4 floats, so left-shift by 2.
cuda::pipeline<cuda::thread_scope_thread> pipe = cuda::make_pipeline();
const auto shape4 = cuda::aligned_size_t<alignof(float4)>(sizeof(float4));
constexpr int loadStride = 2; // load 4 floats, so left-shift by 2.
// Each CTA slides along the 128 x 128 tiles from the top left corner of the matrix to the
// right and down, and selects the next tile to compute. Once there's no such tile,
// all warps in this CTA exit.
for(unsigned int block_pos = blockIdx.x;; block_pos += gridDim.x) {
for (unsigned int block_pos = blockIdx.x;; block_pos += gridDim.x) {
const unsigned int block_tile_i = ((block_pos * BLOCK_ROW_TILES) / N_TILES) * (BLOCK_COL_TILES);
const unsigned int block_tile_j = (block_pos * BLOCK_COL_TILES) % N_TILES;
@@ -419,7 +425,7 @@ __global__ void compute_tf32gemm_async_copy(const float *A, const float *B, cons
}
// This warp's pointer to the C matrix data to copy memory from to shared memory.
const size_t gmem_idx = (block_tile_i + warpId) * M * GLOBAL_MEM_STRIDE + block_tile_j * N;
const size_t gmem_idx = (block_tile_i + warpId) * M * GLOBAL_MEM_STRIDE + block_tile_j * N;
const float *src_gmem_warp_stream_ptr = &C[gmem_idx];
// Stream multiple C tiles to shared memory.
@@ -427,8 +433,9 @@ __global__ void compute_tf32gemm_async_copy(const float *A, const float *B, cons
for (int i = 0; i < N; i++) {
pipe.producer_acquire();
cuda::memcpy_async(&shmem_warp_stream_ptr[(SHMEM_STRIDE * i) + (laneId << loadStride)],
&src_gmem_warp_stream_ptr[(GLOBAL_MEM_STRIDE * i) + (laneId << loadStride)],
shape4, pipe);
&src_gmem_warp_stream_ptr[(GLOBAL_MEM_STRIDE * i) + (laneId << loadStride)],
shape4,
pipe);
pipe.producer_commit();
}
// Now wait for all the above issued 8 batches to complete.
@@ -461,19 +468,22 @@ __global__ void compute_tf32gemm_async_copy(const float *A, const float *B, cons
// Select what warp copies what matrix to shared memory.
// Warps 0-3 copy the A matrix, warps 4-7 copy the B matrix.
const float *warp_ptr = (warpId < (WARPS_PER_BLOCK/2)) ? (&A[block_tile_i * M * K_GLOBAL] + M * K_GLOBAL * (warpId % (WARPS_PER_BLOCK/2)) * 2) :
(&B[block_tile_j * N * K_GLOBAL] + N * K_GLOBAL * (warpId % (WARPS_PER_BLOCK/2)) * 2);
const float *warp_ptr =
(warpId < (WARPS_PER_BLOCK / 2))
? (&A[block_tile_i * M * K_GLOBAL] + M * K_GLOBAL * (warpId % (WARPS_PER_BLOCK / 2)) * 2)
: (&B[block_tile_j * N * K_GLOBAL] + N * K_GLOBAL * (warpId % (WARPS_PER_BLOCK / 2)) * 2);
constexpr int chunksPerLane = ((WARP_SIZE/2) / CHUNK_COPY_LINES_PER_WARP) * 2;
const int laneLoadElem = (laneId % CHUNK_COPY_LINE_LANES) << loadStride;
const int stridePerLaneCopy = (laneId / CHUNK_COPY_LINE_LANES);
constexpr int chunksPerLane = ((WARP_SIZE / 2) / CHUNK_COPY_LINES_PER_WARP) * 2;
const int laneLoadElem = (laneId % CHUNK_COPY_LINE_LANES) << loadStride;
const int stridePerLaneCopy = (laneId / CHUNK_COPY_LINE_LANES);
// Go through the global K dimension by a fixed step at a time.
#pragma unroll
for (int tile_k = 0; tile_k < K_TILES; tile_k += CHUNK_K) {
// Copy slices of the A and B matrices to shared memory.
// The first half of the warps in the CTA copy the A matrix, the rest copy the B matrix.
// As for tf32 MMA M == N we use M for warp 4-7 + shmem_idx_b_off.
size_t shmem_idx = (M * (warpId % (WARPS_PER_BLOCK/2)) * 2) + ((warpId / (WARPS_PER_BLOCK/2)) * shmem_idx_b_off);
size_t shmem_idx =
(M * (warpId % (WARPS_PER_BLOCK / 2)) * 2) + ((warpId / (WARPS_PER_BLOCK / 2)) * shmem_idx_b_off);
// First half of the warp copies the first row / column of the matrix,
// the second half of the warp copies the next.
const float *lane_ptr = (warp_ptr + tile_k * K + stridePerLaneCopy * K_GLOBAL + laneLoadElem);
@@ -482,7 +492,7 @@ __global__ void compute_tf32gemm_async_copy(const float *A, const float *B, cons
shmem_idx += stridePerLaneCopy;
#pragma unroll
for(int i = 0; i < chunksPerLane; i++) {
for (int i = 0; i < chunksPerLane; i++) {
// Copy 16 bytes at once in each lane.
pipe.producer_acquire();
cuda::memcpy_async(&shmem[shmem_idx][laneLoadElem], lane_ptr, shape4, pipe);
@@ -504,8 +514,8 @@ __global__ void compute_tf32gemm_async_copy(const float *A, const float *B, cons
#pragma unroll
for (int i = 0; i < WARP_COL_TILES; i++) {
size_t shmem_idx_a = (warpId / BLOCK_ROW_WARPS) * M * BLOCK_ROW_WARPS + (i * M);
const float *tile_ptr = &shmem[shmem_idx_a][k_step * K];
size_t shmem_idx_a = (warpId / BLOCK_ROW_WARPS) * M * BLOCK_ROW_WARPS + (i * M);
const float *tile_ptr = &shmem[shmem_idx_a][k_step * K];
wmma::load_matrix_sync(a[i], tile_ptr, K * CHUNK_K + SKEW_FLOAT);
@@ -518,13 +528,13 @@ __global__ void compute_tf32gemm_async_copy(const float *A, const float *B, cons
if (i == 0) {
// Load the B matrix fragment once, because it is going to be reused
// against the other A matrix fragments.
size_t shmem_idx_b = shmem_idx_b_off + (WARP_ROW_TILES * N) * (warpId%2) + (j * N);
const float *tile_ptr = &shmem[shmem_idx_b][k_step * K];
size_t shmem_idx_b = shmem_idx_b_off + (WARP_ROW_TILES * N) * (warpId % 2) + (j * N);
const float *tile_ptr = &shmem[shmem_idx_b][k_step * K];
wmma::load_matrix_sync(b[j], tile_ptr, K * CHUNK_K + SKEW_FLOAT);
#pragma unroll
for (int t = 0; t < b[j].num_elements; t++) {
b[j].x[t] = wmma::__float_to_tf32(b[j].x[t]);
b[j].x[t] = wmma::__float_to_tf32(b[j].x[t]);
}
}
@@ -560,8 +570,8 @@ __global__ void compute_tf32gemm_async_copy(const float *A, const float *B, cons
#pragma unroll
for (int i = 0; i < N; i++) {
*((float4*)(dst_gmem_warp_stream_ptr + GLOBAL_MEM_STRIDE * i) + laneId) =
*((float4*)(shmem_warp_stream_ptr + SHMEM_STRIDE * i) + laneId);
*((float4 *)(dst_gmem_warp_stream_ptr + GLOBAL_MEM_STRIDE * i) + laneId) =
*((float4 *)(shmem_warp_stream_ptr + SHMEM_STRIDE * i) + laneId);
}
__syncthreads();
@@ -571,84 +581,90 @@ __global__ void compute_tf32gemm_async_copy(const float *A, const float *B, cons
// Performs an MxNxK tf32 GEMM (C=alpha*A*B + beta*C) assuming:
// 1) Matrices are packed in memory.
// 2) M, N and K are multiples of 16, 16 and 8 respectively.
// 2) M, N and K are multiples of 16, 16 and 8 respectively.
// 3) A is row major, B is column major matrix.
// Note: This is a less performant version of the compute_tf32gemm kernel. It is designed for
// demonstration purposes only to show the CUDA WMMA API use without relying on
// availability of the shared memory.
__global__ void simple_wmma_tf32gemm(float *a, float *b, float *c, float *d, int m_ld, int n_ld, int k_ld, float alpha, float beta)
__global__ void
simple_wmma_tf32gemm(float *a, float *b, float *c, float *d, int m_ld, int n_ld, int k_ld, float alpha, float beta)
{
#if __CUDA_ARCH__ >= 800
// Leading dimensions. Packed with no transpositions.
// Leading dimensions. Packed with no transpositions.
int lda = k_ld;
int ldb = k_ld;
int ldc = n_ld;
// Tile using a 2D grid
int warpM = (blockIdx.x * blockDim.x + threadIdx.x) / warpSize;
int warpN = (blockIdx.y * blockDim.y + threadIdx.y);
// Declare the fragments
wmma::fragment<wmma::matrix_a, M, N, K, wmma::precision::tf32, wmma::row_major> a_frag;
wmma::fragment<wmma::matrix_b, M, N, K, wmma::precision::tf32, wmma::col_major> b_frag;
wmma::fragment<wmma::accumulator, M, N, K, float> acc_frag;
wmma::fragment<wmma::accumulator, M, N, K, float> c_frag;
// Tile using a 2D grid
int warpM = (blockIdx.x * blockDim.x + threadIdx.x) / warpSize;
int warpN = (blockIdx.y * blockDim.y + threadIdx.y);
wmma::fill_fragment(acc_frag, 0.0f);
// Declare the fragments
wmma::fragment<wmma::matrix_a, M, N, K, wmma::precision::tf32, wmma::row_major> a_frag;
wmma::fragment<wmma::matrix_b, M, N, K, wmma::precision::tf32, wmma::col_major> b_frag;
wmma::fragment<wmma::accumulator, M, N, K, float> acc_frag;
wmma::fragment<wmma::accumulator, M, N, K, float> c_frag;
// Loop over k
for (int i = 0; i < k_ld; i += K) {
int aCol = i;
int aRow = warpM * M;
wmma::fill_fragment(acc_frag, 0.0f);
//int bCol = i;
//int bRow = warpN * N;
int bCol = warpN * N;
int bRow = i;
// Loop over k
for (int i = 0; i < k_ld; i += K) {
int aCol = i;
int aRow = warpM * M;
// Bounds checking
if (aRow < m_ld && aCol < k_ld && bRow < k_ld && bCol < n_ld) {
// Load the inputs
wmma::load_matrix_sync(a_frag, a + aCol + aRow * lda, lda);
wmma::load_matrix_sync(b_frag, b + bRow + bCol * ldb, ldb);
#pragma unroll
for (int t = 0; t < a_frag.num_elements; t++) {
a_frag.x[t] = wmma::__float_to_tf32(a_frag.x[t]);
// int bCol = i;
// int bRow = warpN * N;
int bCol = warpN * N;
int bRow = i;
// Bounds checking
if (aRow < m_ld && aCol < k_ld && bRow < k_ld && bCol < n_ld) {
// Load the inputs
wmma::load_matrix_sync(a_frag, a + aCol + aRow * lda, lda);
wmma::load_matrix_sync(b_frag, b + bRow + bCol * ldb, ldb);
#pragma unroll
for (int t = 0; t < a_frag.num_elements; t++) {
a_frag.x[t] = wmma::__float_to_tf32(a_frag.x[t]);
}
#pragma unroll
for (int t = 0; t < b_frag.num_elements; t++) {
b_frag.x[t] = wmma::__float_to_tf32(b_frag.x[t]);
}
// Perform the matrix multiplication
wmma::mma_sync(acc_frag, a_frag, b_frag, acc_frag);
}
}
// Load in the current value of c, scale it by beta, and add this our result scaled by alpha
int cCol = warpN * N;
int cRow = warpM * M;
if (cRow < m_ld && cCol < n_ld) {
wmma::load_matrix_sync(c_frag, c + cCol + cRow * ldc, ldc, wmma::mem_row_major);
for (int i = 0; i < c_frag.num_elements; i++) {
c_frag.x[i] = alpha * acc_frag.x[i] + beta * c_frag.x[i];
}
#pragma unroll
for (int t = 0; t < b_frag.num_elements; t++) {
b_frag.x[t] = wmma::__float_to_tf32(b_frag.x[t]);
}
// Perform the matrix multiplication
wmma::mma_sync(acc_frag, a_frag, b_frag, acc_frag);
}
}
// Load in the current value of c, scale it by beta, and add this our result scaled by alpha
int cCol = warpN * N;
int cRow = warpM * M;
if (cRow < m_ld && cCol < n_ld) {
wmma::load_matrix_sync(c_frag, c + cCol + cRow * ldc, ldc, wmma::mem_row_major);
for(int i=0; i < c_frag.num_elements; i++) {
c_frag.x[i] = alpha * acc_frag.x[i] + beta * c_frag.x[i];
}
// Store the output
wmma::store_matrix_sync(d + cCol + cRow * ldc, c_frag, ldc, wmma::mem_row_major);
}
// Store the output
wmma::store_matrix_sync(d + cCol + cRow * ldc, c_frag, ldc, wmma::mem_row_major);
}
#endif
}
__host__ void matMultiplyOnHost(float *A, float *B, float *C,
float alpha, float beta,
int numARows, int numAColumns,
int numBRows, int numBColumns,
int numCRows, int numCColumns)
__host__ void matMultiplyOnHost(float *A,
float *B,
float *C,
float alpha,
float beta,
int numARows,
int numAColumns,
int numBRows,
int numBColumns,
int numCRows,
int numCColumns)
{
for (int i = 0; i < numCRows; i++) {
for (int j = 0; j < numCColumns; j++) {
@@ -658,7 +674,7 @@ __host__ void matMultiplyOnHost(float *A, float *B, float *C,
temp += A[i * numAColumns + k] * B[j * numBRows + k];
}
C[i*numCColumns + j] = temp * alpha + beta * C[i * numCColumns + j];
C[i * numCColumns + j] = temp * alpha + beta * C[i * numCColumns + j];
}
}
}
@@ -686,16 +702,16 @@ int main(int argc, char **argv)
float *B_h = NULL;
float *C_h = NULL;
#if CPU_DEBUG
float *result_hD = NULL;
float *result_hD = NULL;
float *result_host = NULL;
#endif
A_h = (float*) malloc(sizeof(float) * M_GLOBAL * K_GLOBAL);
B_h = (float*) malloc(sizeof(float) * K_GLOBAL * N_GLOBAL);
C_h = (float*) malloc(sizeof(float) * M_GLOBAL * N_GLOBAL);
A_h = (float *)malloc(sizeof(float) * M_GLOBAL * K_GLOBAL);
B_h = (float *)malloc(sizeof(float) * K_GLOBAL * N_GLOBAL);
C_h = (float *)malloc(sizeof(float) * M_GLOBAL * N_GLOBAL);
#if CPU_DEBUG
result_hD = (float*) malloc(sizeof(float) * M_GLOBAL * N_GLOBAL);
result_host = (float*) malloc(sizeof(float) * M_GLOBAL * N_GLOBAL);
result_hD = (float *)malloc(sizeof(float) * M_GLOBAL * N_GLOBAL);
result_host = (float *)malloc(sizeof(float) * M_GLOBAL * N_GLOBAL);
#endif
float *A = NULL;
@@ -703,10 +719,10 @@ int main(int argc, char **argv)
float *C = NULL;
float *D = NULL;
checkCudaErrors(cudaMalloc((void**)&A, sizeof(float) * M_GLOBAL * K_GLOBAL));
checkCudaErrors(cudaMalloc((void**)&B, sizeof(float) * N_GLOBAL * K_GLOBAL));
checkCudaErrors(cudaMalloc((void**)&C, sizeof(float) * M_GLOBAL * N_GLOBAL));
checkCudaErrors(cudaMalloc((void**)&D, sizeof(float) * M_GLOBAL * N_GLOBAL));
checkCudaErrors(cudaMalloc((void **)&A, sizeof(float) * M_GLOBAL * K_GLOBAL));
checkCudaErrors(cudaMalloc((void **)&B, sizeof(float) * N_GLOBAL * K_GLOBAL));
checkCudaErrors(cudaMalloc((void **)&C, sizeof(float) * M_GLOBAL * N_GLOBAL));
checkCudaErrors(cudaMalloc((void **)&D, sizeof(float) * M_GLOBAL * N_GLOBAL));
assert(((unsigned long long)A) % 128 == 0);
assert(((unsigned long long)B) % 128 == 0);
@@ -734,11 +750,11 @@ int main(int argc, char **argv)
printf("Required shared memory size: %lu Kb\n", SHMEM_SZ / 1024UL);
const float alpha = 1.1f;
const float beta = 1.2f;
const float beta = 1.2f;
cudaEvent_t start, stop;
checkCudaErrors(cudaEventCreate(&start));
checkCudaErrors(cudaEventCreate(&start));
checkCudaErrors(cudaEventCreate(&stop));
checkCudaErrors(cudaEventRecord(start));
@@ -760,26 +776,30 @@ int main(int argc, char **argv)
if ((deviceProp.sharedMemPerMultiprocessor >= SHMEM_SZ) && (selected_kernel != simple_tf32mma_gemm)) {
printf("Computing using high performance kernel = %d - %s\n", selected_kernel, kernelNames[selected_kernel]);
switch (selected_kernel)
{
case tf32mma_shmem_gemm_async_copy :
default:
checkCudaErrors(cudaFuncSetAttribute(compute_tf32gemm_async_copy, cudaFuncAttributeMaxDynamicSharedMemorySize, SHMEM_SZ));
checkKernelErrors((compute_tf32gemm_async_copy<<<deviceProp.multiProcessorCount*2, THREADS_PER_BLOCK, SHMEM_SZ>>>(A, B, C, D, alpha, beta)));
break;
case tf32mma_shmem_gemm :
checkCudaErrors(cudaFuncSetAttribute(compute_tf32gemm, cudaFuncAttributeMaxDynamicSharedMemorySize, SHMEM_SZ));
checkKernelErrors((compute_tf32gemm<<<deviceProp.multiProcessorCount*2, THREADS_PER_BLOCK, SHMEM_SZ>>>(A, B, C, D, alpha, beta)));
break;
switch (selected_kernel) {
case tf32mma_shmem_gemm_async_copy:
default:
checkCudaErrors(cudaFuncSetAttribute(
compute_tf32gemm_async_copy, cudaFuncAttributeMaxDynamicSharedMemorySize, SHMEM_SZ));
checkKernelErrors(
(compute_tf32gemm_async_copy<<<deviceProp.multiProcessorCount * 2, THREADS_PER_BLOCK, SHMEM_SZ>>>(
A, B, C, D, alpha, beta)));
break;
case tf32mma_shmem_gemm:
checkCudaErrors(
cudaFuncSetAttribute(compute_tf32gemm, cudaFuncAttributeMaxDynamicSharedMemorySize, SHMEM_SZ));
checkKernelErrors((compute_tf32gemm<<<deviceProp.multiProcessorCount * 2, THREADS_PER_BLOCK, SHMEM_SZ>>>(
A, B, C, D, alpha, beta)));
break;
}
#if CPU_DEBUG
checkCudaErrors(cudaMemcpy(result_hD, D, sizeof(float)*M_GLOBAL*N_GLOBAL, cudaMemcpyDeviceToHost));
checkCudaErrors(cudaMemcpy(result_hD, D, sizeof(float) * M_GLOBAL * N_GLOBAL, cudaMemcpyDeviceToHost));
#endif
}
else {
dim3 gridDim;
dim3 blockDim;
// blockDim.x must be a multple of warpSize
// 128x4 means we have 16 warps and a block computes a 64x64 output tile
blockDim.x = 128;
@@ -803,11 +823,7 @@ int main(int argc, char **argv)
memcpy(result_host, C_h, sizeof(float) * M_GLOBAL * N_GLOBAL);
matMultiplyOnHost(A_h, B_h, result_host,
alpha, beta,
M_GLOBAL, K_GLOBAL,
K_GLOBAL, N_GLOBAL,
M_GLOBAL, N_GLOBAL);
matMultiplyOnHost(A_h, B_h, result_host, alpha, beta, M_GLOBAL, K_GLOBAL, K_GLOBAL, N_GLOBAL, M_GLOBAL, N_GLOBAL);
for (int i = 0; i < N_GLOBAL * M_GLOBAL; i++) {
if (fabs(result_hD[i] - result_host[i]) > 0.1f) {
@@ -823,15 +839,15 @@ int main(int argc, char **argv)
checkCudaErrors(cudaEventElapsedTime(&milliseconds, start, stop));
printf("Time: %f ms\n", milliseconds);
printf("TFLOPS: %.2f\n", (((double)M_GLOBAL * N_GLOBAL * K_GLOBAL * 2)/(milliseconds/1000.)) / 1e12);
printf("TFLOPS: %.2f\n", (((double)M_GLOBAL * N_GLOBAL * K_GLOBAL * 2) / (milliseconds / 1000.)) / 1e12);
free(A_h);
free(B_h);
free(C_h);
checkCudaErrors(cudaFree((void*)A));
checkCudaErrors(cudaFree((void*)B));
checkCudaErrors(cudaFree((void*)C));
checkCudaErrors(cudaFree((void*)D));
checkCudaErrors(cudaFree((void *)A));
checkCudaErrors(cudaFree((void *)B));
checkCudaErrors(cudaFree((void *)C));
checkCudaErrors(cudaFree((void *)D));
return 0;
}

424
Samples/3_CUDA_Features/warpAggregatedAtomicsCG/warpAggregatedAtomicsCG.cu
View File

@@ -27,301 +27,287 @@
#include <stdio.h>
// includes, project
#include <cooperative_groups.h>
#include <cooperative_groups/reduce.h>
#include <cuda_runtime.h>
#include <helper_cuda.h>
#include <helper_functions.h>
#include <cuda_runtime.h>
#include <cooperative_groups.h>
#include <cooperative_groups/reduce.h>
namespace cg = cooperative_groups;
#define NUM_ELEMS 10000000
#define NUM_ELEMS 10000000
#define NUM_THREADS_PER_BLOCK 512
// warp-aggregated atomic increment
__device__ int atomicAggInc(int *counter) {
cg::coalesced_group active = cg::coalesced_threads();
__device__ int atomicAggInc(int *counter)
{
cg::coalesced_group active = cg::coalesced_threads();
// leader does the update
int res = 0;
if (active.thread_rank() == 0) {
res = atomicAdd(counter, active.size());
}
// leader does the update
int res = 0;
if (active.thread_rank() == 0) {
res = atomicAdd(counter, active.size());
}
// broadcast result
res = active.shfl(res, 0);
// broadcast result
res = active.shfl(res, 0);
// each thread computes its own value
return res + active.thread_rank();
// each thread computes its own value
return res + active.thread_rank();
}
__global__ void filter_arr(int *dst, int *nres, const int *src, int n) {
int id = threadIdx.x + blockIdx.x * blockDim.x;
__global__ void filter_arr(int *dst, int *nres, const int *src, int n)
{
int id = threadIdx.x + blockIdx.x * blockDim.x;
for (int i = id; i < n; i += gridDim.x * blockDim.x) {
if (src[i] > 0) dst[atomicAggInc(nres)] = src[i];
}
for (int i = id; i < n; i += gridDim.x * blockDim.x) {
if (src[i] > 0)
dst[atomicAggInc(nres)] = src[i];
}
}
// warp-aggregated atomic multi bucket increment
#if __CUDA_ARCH__ >= 700
__device__ int atomicAggIncMulti(const int bucket, int *counter)
{
cg::coalesced_group active = cg::coalesced_threads();
// group all threads with same bucket value.
auto labeledGroup = cg::labeled_partition(active, bucket);
cg::coalesced_group active = cg::coalesced_threads();
// group all threads with same bucket value.
auto labeledGroup = cg::labeled_partition(active, bucket);
int res = 0;
if (labeledGroup.thread_rank() == 0)
{
res = atomicAdd(&counter[bucket], labeledGroup.size());
}
int res = 0;
if (labeledGroup.thread_rank() == 0) {
res = atomicAdd(&counter[bucket], labeledGroup.size());
}
// broadcast result
res = labeledGroup.shfl(res, 0);
// broadcast result
res = labeledGroup.shfl(res, 0);
// each thread computes its own value
return res + labeledGroup.thread_rank();
// each thread computes its own value
return res + labeledGroup.thread_rank();
}
#endif
// Places individual value indices into its corresponding buckets.
__global__ void mapToBuckets(const int *srcArr, int *indicesBuckets, int *bucketCounters, const int srcSize, const int numOfBuckets)
__global__ void
mapToBuckets(const int *srcArr, int *indicesBuckets, int *bucketCounters, const int srcSize, const int numOfBuckets)
{
#if __CUDA_ARCH__ >= 700
cg::grid_group grid = cg::this_grid();
cg::grid_group grid = cg::this_grid();
for (int i=grid.thread_rank(); i < srcSize; i += grid.size())
{
const int bucket = srcArr[i];
if (bucket < numOfBuckets)
{
indicesBuckets[atomicAggIncMulti(bucket, bucketCounters)] = i;
for (int i = grid.thread_rank(); i < srcSize; i += grid.size()) {
const int bucket = srcArr[i];
if (bucket < numOfBuckets) {
indicesBuckets[atomicAggIncMulti(bucket, bucketCounters)] = i;
}
}
}
#endif
}
int mapIndicesToBuckets(int *h_srcArr, int *d_srcArr, int numOfBuckets)
{
int *d_indicesBuckets, *d_bucketCounters;
int *cpuBucketCounters = new int[numOfBuckets];
int *h_bucketCounters = new int[numOfBuckets];
int *d_indicesBuckets, *d_bucketCounters;
int *cpuBucketCounters = new int[numOfBuckets];
int *h_bucketCounters = new int[numOfBuckets];
memset(cpuBucketCounters, 0, sizeof(int)*numOfBuckets);
// Initialize each bucket counters.
for (int i = 0; i < numOfBuckets; i++)
{
h_bucketCounters[i] = i*NUM_ELEMS;
}
checkCudaErrors(cudaMalloc(&d_indicesBuckets, sizeof(int) * NUM_ELEMS * numOfBuckets));
checkCudaErrors(cudaMalloc(&d_bucketCounters, sizeof(int) * numOfBuckets));
checkCudaErrors(cudaMemcpy(d_bucketCounters, h_bucketCounters, sizeof(int)*numOfBuckets, cudaMemcpyHostToDevice));
dim3 dimBlock(NUM_THREADS_PER_BLOCK, 1, 1);
dim3 dimGrid((NUM_ELEMS / NUM_THREADS_PER_BLOCK), 1, 1);
mapToBuckets<<<dimGrid, dimBlock>>>(d_srcArr, d_indicesBuckets, d_bucketCounters, NUM_ELEMS, numOfBuckets);
checkCudaErrors(cudaMemcpy(h_bucketCounters, d_bucketCounters, sizeof(int)*numOfBuckets, cudaMemcpyDeviceToHost));
for (int i=0; i < NUM_ELEMS; i++)
{
cpuBucketCounters[h_srcArr[i]]++;
}
bool allMatch = true;
int finalElems = 0;
for (int i=0; i < numOfBuckets; i++)
{
finalElems += (h_bucketCounters[i] - i*NUM_ELEMS);
if (cpuBucketCounters[i] != (h_bucketCounters[i] - i*NUM_ELEMS))
{
allMatch = false;
break;
memset(cpuBucketCounters, 0, sizeof(int) * numOfBuckets);
// Initialize each bucket counters.
for (int i = 0; i < numOfBuckets; i++) {
h_bucketCounters[i] = i * NUM_ELEMS;
}
}
if (!allMatch && finalElems != NUM_ELEMS)
{
return EXIT_FAILURE;
}
return EXIT_SUCCESS;
checkCudaErrors(cudaMalloc(&d_indicesBuckets, sizeof(int) * NUM_ELEMS * numOfBuckets));
checkCudaErrors(cudaMalloc(&d_bucketCounters, sizeof(int) * numOfBuckets));
checkCudaErrors(cudaMemcpy(d_bucketCounters, h_bucketCounters, sizeof(int) * numOfBuckets, cudaMemcpyHostToDevice));
dim3 dimBlock(NUM_THREADS_PER_BLOCK, 1, 1);
dim3 dimGrid((NUM_ELEMS / NUM_THREADS_PER_BLOCK), 1, 1);
mapToBuckets<<<dimGrid, dimBlock>>>(d_srcArr, d_indicesBuckets, d_bucketCounters, NUM_ELEMS, numOfBuckets);
checkCudaErrors(cudaMemcpy(h_bucketCounters, d_bucketCounters, sizeof(int) * numOfBuckets, cudaMemcpyDeviceToHost));
for (int i = 0; i < NUM_ELEMS; i++) {
cpuBucketCounters[h_srcArr[i]]++;
}
bool allMatch = true;
int finalElems = 0;
for (int i = 0; i < numOfBuckets; i++) {
finalElems += (h_bucketCounters[i] - i * NUM_ELEMS);
if (cpuBucketCounters[i] != (h_bucketCounters[i] - i * NUM_ELEMS)) {
allMatch = false;
break;
}
}
if (!allMatch && finalElems != NUM_ELEMS) {
return EXIT_FAILURE;
}
return EXIT_SUCCESS;
}
// Warp-aggregated atomic Max in multi bucket
#if __CUDA_ARCH__ >= 700
__device__ void atomicAggMaxMulti(const int bucket, int *counter, const int valueForMax)
{
cg::coalesced_group active = cg::coalesced_threads();
// group all threads with same bucket value.
auto labeledGroup = cg::labeled_partition(active, bucket);
cg::coalesced_group active = cg::coalesced_threads();
// group all threads with same bucket value.
auto labeledGroup = cg::labeled_partition(active, bucket);
const int maxValueInGroup = cg::reduce(labeledGroup, valueForMax, cg::greater<int>());
const int maxValueInGroup = cg::reduce(labeledGroup, valueForMax, cg::greater<int>());
if (labeledGroup.thread_rank() == 0)
{
atomicMax(&counter[bucket], maxValueInGroup);
}
if (labeledGroup.thread_rank() == 0) {
atomicMax(&counter[bucket], maxValueInGroup);
}
}
#endif
// Performs max calculation in each buckets.
__global__ void calculateMaxInEachBuckets(const int *srcArr, const int *valueInBuckets, int *bucketsMax, const int srcSize, const int numOfBuckets)
__global__ void calculateMaxInEachBuckets(const int *srcArr,
const int *valueInBuckets,
int *bucketsMax,
const int srcSize,
const int numOfBuckets)
{
#if __CUDA_ARCH__ >= 700
cg::grid_group grid = cg::this_grid();
cg::grid_group grid = cg::this_grid();
for (int i=grid.thread_rank(); i < srcSize; i += grid.size())
{
const int bucket = srcArr[i];
if (bucket < numOfBuckets)
{
atomicAggMaxMulti(bucket, bucketsMax, valueInBuckets[i]);
for (int i = grid.thread_rank(); i < srcSize; i += grid.size()) {
const int bucket = srcArr[i];
if (bucket < numOfBuckets) {
atomicAggMaxMulti(bucket, bucketsMax, valueInBuckets[i]);
}
}
}
#endif
}
int calculateMaxInBuckets(int *h_srcArr, int *d_srcArr, int numOfBuckets)
{
int *d_valueInBuckets, *d_bucketsMax;
int *h_valueInBuckets = new int[NUM_ELEMS];
int *cpuBucketsMax = new int[numOfBuckets];
int *h_bucketsMax = new int[numOfBuckets];
int *d_valueInBuckets, *d_bucketsMax;
int *h_valueInBuckets = new int[NUM_ELEMS];
int *cpuBucketsMax = new int[numOfBuckets];
int *h_bucketsMax = new int[numOfBuckets];
memset(cpuBucketsMax, 0, sizeof(int) * numOfBuckets);
memset(cpuBucketsMax, 0, sizeof(int) * numOfBuckets);
// Here we create values which is assumed to correspond to each
// buckets of srcArr at same array index.
for (int i=0; i < NUM_ELEMS; i++)
{
h_valueInBuckets[i] = rand();
}
checkCudaErrors(cudaMalloc(&d_valueInBuckets, sizeof(int) * NUM_ELEMS));
checkCudaErrors(cudaMalloc(&d_bucketsMax, sizeof(int) * numOfBuckets));
checkCudaErrors(cudaMemset(d_bucketsMax, 0, sizeof(int) * numOfBuckets));
checkCudaErrors(cudaMemcpy(d_valueInBuckets, h_valueInBuckets, sizeof(int) * NUM_ELEMS, cudaMemcpyHostToDevice));
dim3 dimBlock(NUM_THREADS_PER_BLOCK, 1, 1);
dim3 dimGrid((NUM_ELEMS / NUM_THREADS_PER_BLOCK), 1, 1);
calculateMaxInEachBuckets<<<dimGrid, dimBlock>>>(d_srcArr, d_valueInBuckets, d_bucketsMax, NUM_ELEMS, numOfBuckets);
checkCudaErrors(cudaMemcpy(h_bucketsMax, d_bucketsMax, sizeof(int) * numOfBuckets, cudaMemcpyDeviceToHost));
for (int i = 0; i < NUM_ELEMS; i++)
{
if (cpuBucketsMax[h_srcArr[i]] < h_valueInBuckets[i])
{
cpuBucketsMax[h_srcArr[i]] = h_valueInBuckets[i];
// Here we create values which is assumed to correspond to each
// buckets of srcArr at same array index.
for (int i = 0; i < NUM_ELEMS; i++) {
h_valueInBuckets[i] = rand();
}
}
bool allMatch = true;
int finalElems = 0;
for (int i=0; i < numOfBuckets; i++)
{
if (cpuBucketsMax[i] != h_bucketsMax[i])
{
allMatch = false;
printf("CPU i=%d max = %d mismatches GPU max = %d\n", i, cpuBucketsMax[i], h_bucketsMax[i]);
break;
checkCudaErrors(cudaMalloc(&d_valueInBuckets, sizeof(int) * NUM_ELEMS));
checkCudaErrors(cudaMalloc(&d_bucketsMax, sizeof(int) * numOfBuckets));
checkCudaErrors(cudaMemset(d_bucketsMax, 0, sizeof(int) * numOfBuckets));
checkCudaErrors(cudaMemcpy(d_valueInBuckets, h_valueInBuckets, sizeof(int) * NUM_ELEMS, cudaMemcpyHostToDevice));
dim3 dimBlock(NUM_THREADS_PER_BLOCK, 1, 1);
dim3 dimGrid((NUM_ELEMS / NUM_THREADS_PER_BLOCK), 1, 1);
calculateMaxInEachBuckets<<<dimGrid, dimBlock>>>(d_srcArr, d_valueInBuckets, d_bucketsMax, NUM_ELEMS, numOfBuckets);
checkCudaErrors(cudaMemcpy(h_bucketsMax, d_bucketsMax, sizeof(int) * numOfBuckets, cudaMemcpyDeviceToHost));
for (int i = 0; i < NUM_ELEMS; i++) {
if (cpuBucketsMax[h_srcArr[i]] < h_valueInBuckets[i]) {
cpuBucketsMax[h_srcArr[i]] = h_valueInBuckets[i];
}
}
}
if (allMatch)
{
printf("CPU max matches GPU max\n");
}
delete[] h_valueInBuckets;
delete[] cpuBucketsMax;
delete[] h_bucketsMax;
checkCudaErrors(cudaFree(d_valueInBuckets));
checkCudaErrors(cudaFree(d_bucketsMax));
bool allMatch = true;
int finalElems = 0;
for (int i = 0; i < numOfBuckets; i++) {
if (cpuBucketsMax[i] != h_bucketsMax[i]) {
allMatch = false;
printf("CPU i=%d max = %d mismatches GPU max = %d\n", i, cpuBucketsMax[i], h_bucketsMax[i]);
break;
}
}
if (allMatch) {
printf("CPU max matches GPU max\n");
}
if (!allMatch && finalElems != NUM_ELEMS)
{
return EXIT_FAILURE;
}
delete[] h_valueInBuckets;
delete[] cpuBucketsMax;
delete[] h_bucketsMax;
checkCudaErrors(cudaFree(d_valueInBuckets));
checkCudaErrors(cudaFree(d_bucketsMax));
return EXIT_SUCCESS;
if (!allMatch && finalElems != NUM_ELEMS) {
return EXIT_FAILURE;
}
return EXIT_SUCCESS;
}
int main(int argc, char **argv) {
int *data_to_filter, *filtered_data, nres = 0;
int *d_data_to_filter, *d_filtered_data, *d_nres;
int main(int argc, char **argv)
{
int *data_to_filter, *filtered_data, nres = 0;
int *d_data_to_filter, *d_filtered_data, *d_nres;
int numOfBuckets = 5;
int numOfBuckets = 5;
data_to_filter = reinterpret_cast<int *>(malloc(sizeof(int) * NUM_ELEMS));
data_to_filter = reinterpret_cast<int *>(malloc(sizeof(int) * NUM_ELEMS));
// Generate input data.
for (int i = 0; i < NUM_ELEMS; i++) {
data_to_filter[i] = rand() % numOfBuckets;
}
int devId = findCudaDevice(argc, (const char **)argv);
checkCudaErrors(cudaMalloc(&d_data_to_filter, sizeof(int) * NUM_ELEMS));
checkCudaErrors(cudaMalloc(&d_filtered_data, sizeof(int) * NUM_ELEMS));
checkCudaErrors(cudaMalloc(&d_nres, sizeof(int)));
checkCudaErrors(cudaMemcpy(d_data_to_filter, data_to_filter,
sizeof(int) * NUM_ELEMS, cudaMemcpyHostToDevice));
checkCudaErrors(cudaMemset(d_nres, 0, sizeof(int)));
dim3 dimBlock(NUM_THREADS_PER_BLOCK, 1, 1);
dim3 dimGrid((NUM_ELEMS / NUM_THREADS_PER_BLOCK) + 1, 1, 1);
filter_arr<<<dimGrid, dimBlock>>>(d_filtered_data, d_nres, d_data_to_filter,
NUM_ELEMS);
checkCudaErrors(
cudaMemcpy(&nres, d_nres, sizeof(int), cudaMemcpyDeviceToHost));
filtered_data = reinterpret_cast<int *>(malloc(sizeof(int) * nres));
checkCudaErrors(cudaMemcpy(filtered_data, d_filtered_data, sizeof(int) * nres,
cudaMemcpyDeviceToHost));
int *host_filtered_data =
reinterpret_cast<int *>(malloc(sizeof(int) * NUM_ELEMS));
// Generate host output with host filtering code.
int host_flt_count = 0;
for (int i = 0; i < NUM_ELEMS; i++) {
if (data_to_filter[i] > 0) {
host_filtered_data[host_flt_count++] = data_to_filter[i];
// Generate input data.
for (int i = 0; i < NUM_ELEMS; i++) {
data_to_filter[i] = rand() % numOfBuckets;
}
}
int major = 0;
checkCudaErrors(cudaDeviceGetAttribute(&major, cudaDevAttrComputeCapabilityMajor, devId));
int devId = findCudaDevice(argc, (const char **)argv);
int mapIndicesToBucketsStatus = EXIT_SUCCESS;
int calculateMaxInBucketsStatus = EXIT_SUCCESS;
// atomicAggIncMulti & atomicAggMaxMulti require a GPU of Volta (SM7X) architecture or higher,
// so that it can take advantage of the new MATCH capability of Volta hardware
if (major >= 7) {
mapIndicesToBucketsStatus = mapIndicesToBuckets(data_to_filter, d_data_to_filter, numOfBuckets);
calculateMaxInBucketsStatus = calculateMaxInBuckets(data_to_filter, d_data_to_filter, numOfBuckets);
}
checkCudaErrors(cudaMalloc(&d_data_to_filter, sizeof(int) * NUM_ELEMS));
checkCudaErrors(cudaMalloc(&d_filtered_data, sizeof(int) * NUM_ELEMS));
checkCudaErrors(cudaMalloc(&d_nres, sizeof(int)));
printf("\nWarp Aggregated Atomics %s \n",
(host_flt_count == nres) && (mapIndicesToBucketsStatus == EXIT_SUCCESS) &&
(calculateMaxInBucketsStatus == EXIT_SUCCESS) ? "PASSED" : "FAILED");
checkCudaErrors(cudaMemcpy(d_data_to_filter, data_to_filter, sizeof(int) * NUM_ELEMS, cudaMemcpyHostToDevice));
checkCudaErrors(cudaMemset(d_nres, 0, sizeof(int)));
checkCudaErrors(cudaFree(d_data_to_filter));
checkCudaErrors(cudaFree(d_filtered_data));
checkCudaErrors(cudaFree(d_nres));
free(data_to_filter);
free(filtered_data);
free(host_filtered_data);
dim3 dimBlock(NUM_THREADS_PER_BLOCK, 1, 1);
dim3 dimGrid((NUM_ELEMS / NUM_THREADS_PER_BLOCK) + 1, 1, 1);
filter_arr<<<dimGrid, dimBlock>>>(d_filtered_data, d_nres, d_data_to_filter, NUM_ELEMS);
checkCudaErrors(cudaMemcpy(&nres, d_nres, sizeof(int), cudaMemcpyDeviceToHost));
filtered_data = reinterpret_cast<int *>(malloc(sizeof(int) * nres));
checkCudaErrors(cudaMemcpy(filtered_data, d_filtered_data, sizeof(int) * nres, cudaMemcpyDeviceToHost));
int *host_filtered_data = reinterpret_cast<int *>(malloc(sizeof(int) * NUM_ELEMS));
// Generate host output with host filtering code.
int host_flt_count = 0;
for (int i = 0; i < NUM_ELEMS; i++) {
if (data_to_filter[i] > 0) {
host_filtered_data[host_flt_count++] = data_to_filter[i];
}
}
int major = 0;
checkCudaErrors(cudaDeviceGetAttribute(&major, cudaDevAttrComputeCapabilityMajor, devId));
int mapIndicesToBucketsStatus = EXIT_SUCCESS;
int calculateMaxInBucketsStatus = EXIT_SUCCESS;
// atomicAggIncMulti & atomicAggMaxMulti require a GPU of Volta (SM7X) architecture or higher,
// so that it can take advantage of the new MATCH capability of Volta hardware
if (major >= 7) {
mapIndicesToBucketsStatus = mapIndicesToBuckets(data_to_filter, d_data_to_filter, numOfBuckets);
calculateMaxInBucketsStatus = calculateMaxInBuckets(data_to_filter, d_data_to_filter, numOfBuckets);
}
printf("\nWarp Aggregated Atomics %s \n",
(host_flt_count == nres) && (mapIndicesToBucketsStatus == EXIT_SUCCESS)
&& (calculateMaxInBucketsStatus == EXIT_SUCCESS)
? "PASSED"
: "FAILED");
checkCudaErrors(cudaFree(d_data_to_filter));
checkCudaErrors(cudaFree(d_filtered_data));
checkCudaErrors(cudaFree(d_nres));
free(data_to_filter);
free(filtered_data);
free(host_filtered_data);
}