NASP/cuda-samples
84
0
Fork 0
mirror of https://github.com/NVIDIA/cuda-samples.git synced 2026-10-13 00:08:26 +08:00
Code Issues Packages Projects Releases Wiki Activity

CUDA 13.2 samples update (#432)

Browse Source 
- Added Python samples for CUDA Python 1.0 release
- Renamed top-level `Samples` directory to `cpp` to accommodate Python samples.
This commit is contained in:
Dheemanth
2026-05-13 15:13:18 -07:00
committed by GitHub
parent a4526d5229
commit aeab82ff30
2072 changed files with 15077 additions and 1970 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
cpp/5_Domain_Specific/convolutionFFT2D/.vscode/c_cpp_properties.json vendored Normal file
View File

@@ -0,0 +1,18 @@
{
"configurations": [
{
"name": "Linux",
"includePath": [
"${workspaceFolder}/**",
"${workspaceFolder}/../../../Common"
],
"defines": [],
"compilerPath": "/usr/local/cuda/bin/nvcc",
"cStandard": "gnu17",
"cppStandard": "gnu++14",
"intelliSenseMode": "linux-gcc-x64",
"configurationProvider": "ms-vscode.makefile-tools"
}
],
"version": 4
}

7
cpp/5_Domain_Specific/convolutionFFT2D/.vscode/extensions.json vendored Normal file
View File

@@ -0,0 +1,7 @@
{
"recommendations": [
"nvidia.nsight-vscode-edition",
"ms-vscode.cpptools",
"ms-vscode.makefile-tools"
]
}

42
cpp/5_Domain_Specific/convolutionFFT2D/CMakeLists.txt Normal file
View File

@@ -0,0 +1,42 @@
cmake_minimum_required(VERSION 3.20)
list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/../../../cmake/Modules")
project(convolutionFFT2D LANGUAGES C CXX CUDA)
find_package(CUDAToolkit REQUIRED)
set(CMAKE_POSITION_INDEPENDENT_CODE ON)
set(CMAKE_CUDA_ARCHITECTURES 75 80 86 87 89 90 100 110 120)
set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Wno-deprecated-gpu-targets")
if(ENABLE_CUDA_DEBUG)
set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -G") # enable cuda-gdb (may significantly affect performance on some targets)
else()
set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -lineinfo") # add line information to all builds for debug tools (exclusive to -G option)
endif()
# Include directories and libraries
include_directories(../../../Common)
# Source file
# Add target for convolutionFFT2D
add_executable(convolutionFFT2D convolutionFFT2D.cu convolutionFFT2D_gold.cpp main.cpp)
target_compile_options(convolutionFFT2D PRIVATE $<$<COMPILE_LANGUAGE:CUDA>:--extended-lambda>)
target_compile_features(convolutionFFT2D PRIVATE cxx_std_17 cuda_std_17)
set_target_properties(convolutionFFT2D PROPERTIES CUDA_SEPARABLE_COMPILATION ON)
target_include_directories(convolutionFFT2D PRIVATE
${CUDAToolkit_INCLUDE_DIRS}
)
target_link_libraries(convolutionFFT2D PUBLIC
CUDA::cufft
)
# Include installation configuration
include(${CMAKE_CURRENT_SOURCE_DIR}/../../../cmake/InstallSamples.cmake)
setup_samples_install()

36
cpp/5_Domain_Specific/convolutionFFT2D/README.md Normal file
View File

@@ -0,0 +1,36 @@
# convolutionFFT2D - FFT-Based 2D Convolution
## Description
This sample demonstrates how 2D convolutions with very large kernel sizes can be efficiently implemented using FFT transformations.
## Key Concepts
Image Processing, CUFFT Library
## Supported SM Architectures
[SM 5.0 ](https://developer.nvidia.com/cuda-gpus) [SM 5.2 ](https://developer.nvidia.com/cuda-gpus) [SM 5.3 ](https://developer.nvidia.com/cuda-gpus) [SM 6.0 ](https://developer.nvidia.com/cuda-gpus) [SM 6.1 ](https://developer.nvidia.com/cuda-gpus) [SM 7.0 ](https://developer.nvidia.com/cuda-gpus) [SM 7.2 ](https://developer.nvidia.com/cuda-gpus) [SM 7.5 ](https://developer.nvidia.com/cuda-gpus) [SM 8.0 ](https://developer.nvidia.com/cuda-gpus) [SM 8.6 ](https://developer.nvidia.com/cuda-gpus) [SM 8.7 ](https://developer.nvidia.com/cuda-gpus) [SM 8.9 ](https://developer.nvidia.com/cuda-gpus) [SM 9.0 ](https://developer.nvidia.com/cuda-gpus)
## Supported OSes
Linux, Windows
## Supported CPU Architecture
x86_64, armv7l
## CUDA APIs involved
### [CUDA Runtime API](http://docs.nvidia.com/cuda/cuda-runtime-api/index.html)
cudaMemcpy, cudaFree, cudaDestroyTextureObject, cudaDeviceSynchronize, cudaCreateTextureObject, cudaMemset, cudaMalloc
## Dependencies needed to build/run
[CUFFT](../../../README.md#cufft)
## Prerequisites
Download and install the [CUDA Toolkit](https://developer.nvidia.com/cuda-downloads) for your corresponding platform.
Make sure the dependencies mentioned in [Dependencies]() section above are installed.
## References (for more details)

354
cpp/5_Domain_Specific/convolutionFFT2D/convolutionFFT2D.cu Normal file
View File

@@ -0,0 +1,354 @@
/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of NVIDIA CORPORATION nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
* OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include <assert.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
// Other helpers
#include <helper_cuda.h>
// Project includes
#include "convolutionFFT2D.cuh"
#include "convolutionFFT2D_common.h"
////////////////////////////////////////////////////////////////////////////////
/// Position convolution kernel center at (0, 0) in the image
////////////////////////////////////////////////////////////////////////////////
extern "C" void
padKernel(float *d_Dst, float *d_Src, int fftH, int fftW, int kernelH, int kernelW, int kernelY, int kernelX)
{
assert(d_Src != d_Dst);
dim3 threads(32, 8);
dim3 grid(iDivUp(kernelW, threads.x), iDivUp(kernelH, threads.y));
SET_FLOAT_BASE;
#if (USE_TEXTURE)
cudaTextureObject_t texFloat;
cudaResourceDesc texRes;
memset(&texRes, 0, sizeof(cudaResourceDesc));
texRes.resType = cudaResourceTypeLinear;
texRes.res.linear.devPtr = d_Src;
texRes.res.linear.sizeInBytes = sizeof(float) * kernelH * kernelW;
texRes.res.linear.desc = cudaCreateChannelDesc<float>();
cudaTextureDesc texDescr;
memset(&texDescr, 0, sizeof(cudaTextureDesc));
texDescr.normalizedCoords = false;
texDescr.filterMode = cudaFilterModeLinear;
texDescr.addressMode[0] = cudaAddressModeWrap;
texDescr.readMode = cudaReadModeElementType;
checkCudaErrors(cudaCreateTextureObject(&texFloat, &texRes, &texDescr, NULL));
#endif
padKernel_kernel<<<grid, threads>>>(d_Dst,
d_Src,
fftH,
fftW,
kernelH,
kernelW,
kernelY,
kernelX
#if (USE_TEXTURE)
,
texFloat
#endif
);
getLastCudaError("padKernel_kernel<<<>>> execution failed\n");
#if (USE_TEXTURE)
checkCudaErrors(cudaDestroyTextureObject(texFloat));
#endif
}
////////////////////////////////////////////////////////////////////////////////
// Prepare data for "pad to border" addressing mode
////////////////////////////////////////////////////////////////////////////////
extern "C" void padDataClampToBorder(float *d_Dst,
float *d_Src,
int fftH,
int fftW,
int dataH,
int dataW,
int kernelW,
int kernelH,
int kernelY,
int kernelX)
{
assert(d_Src != d_Dst);
dim3 threads(32, 8);
dim3 grid(iDivUp(fftW, threads.x), iDivUp(fftH, threads.y));
#if (USE_TEXTURE)
cudaTextureObject_t texFloat;
cudaResourceDesc texRes;
memset(&texRes, 0, sizeof(cudaResourceDesc));
texRes.resType = cudaResourceTypeLinear;
texRes.res.linear.devPtr = d_Src;
texRes.res.linear.sizeInBytes = sizeof(float) * dataH * dataW;
texRes.res.linear.desc = cudaCreateChannelDesc<float>();
cudaTextureDesc texDescr;
memset(&texDescr, 0, sizeof(cudaTextureDesc));
texDescr.normalizedCoords = false;
texDescr.filterMode = cudaFilterModeLinear;
texDescr.addressMode[0] = cudaAddressModeWrap;
texDescr.readMode = cudaReadModeElementType;
checkCudaErrors(cudaCreateTextureObject(&texFloat, &texRes, &texDescr, NULL));
#endif
padDataClampToBorder_kernel<<<grid, threads>>>(d_Dst,
d_Src,
fftH,
fftW,
dataH,
dataW,
kernelH,
kernelW,
kernelY,
kernelX
#if (USE_TEXTURE)
,
texFloat
#endif
);
getLastCudaError("padDataClampToBorder_kernel<<<>>> execution failed\n");
#if (USE_TEXTURE)
checkCudaErrors(cudaDestroyTextureObject(texFloat));
#endif
}
////////////////////////////////////////////////////////////////////////////////
// Modulate Fourier image of padded data by Fourier image of padded kernel
// and normalize by FFT size
////////////////////////////////////////////////////////////////////////////////
extern "C" void modulateAndNormalize(fComplex *d_Dst, fComplex *d_Src, int fftH, int fftW, int padding)
{
assert(fftW % 2 == 0);
const int dataSize = fftH * (fftW / 2 + padding);
modulateAndNormalize_kernel<<<iDivUp(dataSize, 256), 256>>>(d_Dst, d_Src, dataSize, 1.0f / (float)(fftW * fftH));
getLastCudaError("modulateAndNormalize() execution failed\n");
}
////////////////////////////////////////////////////////////////////////////////
// 2D R2C / C2R post/preprocessing kernels
////////////////////////////////////////////////////////////////////////////////
static const double PI = 3.1415926535897932384626433832795;
static const uint BLOCKDIM = 256;
extern "C" void spPostprocess2D(void *d_Dst, void *d_Src, uint DY, uint DX, uint padding, int dir)
{
assert(d_Src != d_Dst);
assert(DX % 2 == 0);
#if (POWER_OF_TWO)
uint log2DX, log2DY;
uint factorizationRemX = factorRadix2(log2DX, DX);
uint factorizationRemY = factorRadix2(log2DY, DY);
assert(factorizationRemX == 1 && factorizationRemY == 1);
#endif
const uint threadCount = DY * (DX / 2);
const double phaseBase = dir * PI / (double)DX;
#if (USE_TEXTURE)
cudaTextureObject_t texComplex;
cudaResourceDesc texRes;
memset(&texRes, 0, sizeof(cudaResourceDesc));
texRes.resType = cudaResourceTypeLinear;
texRes.res.linear.devPtr = d_Src;
texRes.res.linear.sizeInBytes = sizeof(fComplex) * DY * (DX + padding);
texRes.res.linear.desc = cudaCreateChannelDesc<fComplex>();
cudaTextureDesc texDescr;
memset(&texDescr, 0, sizeof(cudaTextureDesc));
texDescr.normalizedCoords = false;
texDescr.filterMode = cudaFilterModeLinear;
texDescr.addressMode[0] = cudaAddressModeWrap;
texDescr.readMode = cudaReadModeElementType;
checkCudaErrors(cudaCreateTextureObject(&texComplex, &texRes, &texDescr, NULL));
#endif
spPostprocess2D_kernel<<<iDivUp(threadCount, BLOCKDIM), BLOCKDIM>>>((fComplex *)d_Dst,
(fComplex *)d_Src,
DY,
DX,
threadCount,
padding,
(float)phaseBase
#if (USE_TEXTURE)
,
texComplex
#endif
);
getLastCudaError("spPostprocess2D_kernel<<<>>> execution failed\n");
#if (USE_TEXTURE)
checkCudaErrors(cudaDestroyTextureObject(texComplex));
#endif
}
extern "C" void spPreprocess2D(void *d_Dst, void *d_Src, uint DY, uint DX, uint padding, int dir)
{
assert(d_Src != d_Dst);
assert(DX % 2 == 0);
#if (POWER_OF_TWO)
uint log2DX, log2DY;
uint factorizationRemX = factorRadix2(log2DX, DX);
uint factorizationRemY = factorRadix2(log2DY, DY);
assert(factorizationRemX == 1 && factorizationRemY == 1);
#endif
const uint threadCount = DY * (DX / 2);
const double phaseBase = -dir * PI / (double)DX;
#if (USE_TEXTURE)
cudaTextureObject_t texComplex;
cudaResourceDesc texRes;
memset(&texRes, 0, sizeof(cudaResourceDesc));
texRes.resType = cudaResourceTypeLinear;
texRes.res.linear.devPtr = d_Src;
texRes.res.linear.sizeInBytes = sizeof(fComplex) * DY * (DX + padding);
texRes.res.linear.desc = cudaCreateChannelDesc<fComplex>();
cudaTextureDesc texDescr;
memset(&texDescr, 0, sizeof(cudaTextureDesc));
texDescr.normalizedCoords = false;
texDescr.filterMode = cudaFilterModeLinear;
texDescr.addressMode[0] = cudaAddressModeWrap;
texDescr.readMode = cudaReadModeElementType;
checkCudaErrors(cudaCreateTextureObject(&texComplex, &texRes, &texDescr, NULL));
#endif
spPreprocess2D_kernel<<<iDivUp(threadCount, BLOCKDIM), BLOCKDIM>>>((fComplex *)d_Dst,
(fComplex *)d_Src,
DY,
DX,
threadCount,
padding,
(float)phaseBase
#if (USE_TEXTURE)
,
texComplex
#endif
);
getLastCudaError("spPreprocess2D_kernel<<<>>> execution failed\n");
#if (USE_TEXTURE)
checkCudaErrors(cudaDestroyTextureObject(texComplex));
#endif
}
////////////////////////////////////////////////////////////////////////////////
// Combined spPostprocess2D + modulateAndNormalize + spPreprocess2D
////////////////////////////////////////////////////////////////////////////////
extern "C" void spProcess2D(void *d_Dst, void *d_SrcA, void *d_SrcB, uint DY, uint DX, int dir)
{
assert(DY % 2 == 0);
#if (POWER_OF_TWO)
uint log2DX, log2DY;
uint factorizationRemX = factorRadix2(log2DX, DX);
uint factorizationRemY = factorRadix2(log2DY, DY);
assert(factorizationRemX == 1 && factorizationRemY == 1);
#endif
const uint threadCount = (DY / 2) * DX;
const double phaseBase = dir * PI / (double)DX;
#if (USE_TEXTURE)
cudaTextureObject_t texComplexA, texComplexB;
cudaResourceDesc texRes;
memset(&texRes, 0, sizeof(cudaResourceDesc));
texRes.resType = cudaResourceTypeLinear;
texRes.res.linear.devPtr = d_SrcA;
texRes.res.linear.sizeInBytes = sizeof(fComplex) * DY * DX;
texRes.res.linear.desc = cudaCreateChannelDesc<fComplex>();
cudaTextureDesc texDescr;
memset(&texDescr, 0, sizeof(cudaTextureDesc));
texDescr.normalizedCoords = false;
texDescr.filterMode = cudaFilterModeLinear;
texDescr.addressMode[0] = cudaAddressModeWrap;
texDescr.readMode = cudaReadModeElementType;
checkCudaErrors(cudaCreateTextureObject(&texComplexA, &texRes, &texDescr, NULL));
memset(&texRes, 0, sizeof(cudaResourceDesc));
texRes.resType = cudaResourceTypeLinear;
texRes.res.linear.devPtr = d_SrcB;
texRes.res.linear.sizeInBytes = sizeof(fComplex) * DY * DX;
texRes.res.linear.desc = cudaCreateChannelDesc<fComplex>();
memset(&texDescr, 0, sizeof(cudaTextureDesc));
texDescr.normalizedCoords = false;
texDescr.filterMode = cudaFilterModeLinear;
texDescr.addressMode[0] = cudaAddressModeWrap;
texDescr.readMode = cudaReadModeElementType;
checkCudaErrors(cudaCreateTextureObject(&texComplexB, &texRes, &texDescr, NULL));
#endif
spProcess2D_kernel<<<iDivUp(threadCount, BLOCKDIM), BLOCKDIM>>>((fComplex *)d_Dst,
(fComplex *)d_SrcA,
(fComplex *)d_SrcB,
DY,
DX,
threadCount,
(float)phaseBase,
0.5f / (float)(DY * DX)
#if (USE_TEXTURE)
,
texComplexA,
texComplexB
#endif
);
getLastCudaError("spProcess2D_kernel<<<>>> execution failed\n");
#if (USE_TEXTURE)
checkCudaErrors(cudaDestroyTextureObject(texComplexA));
checkCudaErrors(cudaDestroyTextureObject(texComplexB));
#endif
}

454
cpp/5_Domain_Specific/convolutionFFT2D/convolutionFFT2D.cuh Normal file
View File

@@ -0,0 +1,454 @@
/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of NVIDIA CORPORATION nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
* OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#define USE_TEXTURE 1
#define POWER_OF_TWO 1
#if (USE_TEXTURE)
#define LOAD_FLOAT(i) tex1Dfetch<float>(texFloat, i)
#define SET_FLOAT_BASE
#else
#define LOAD_FLOAT(i) d_Src[i]
#define SET_FLOAT_BASE
#endif
#include "convolutionFFT2D_common.h"
////////////////////////////////////////////////////////////////////////////////
/// Position convolution kernel center at (0, 0) in the image
////////////////////////////////////////////////////////////////////////////////
__global__ void padKernel_kernel(float *d_Dst,
float *d_Src,
int fftH,
int fftW,
int kernelH,
int kernelW,
int kernelY,
int kernelX
#if (USE_TEXTURE)
,
cudaTextureObject_t texFloat
#endif
)
{
const int y = blockDim.y * blockIdx.y + threadIdx.y;
const int x = blockDim.x * blockIdx.x + threadIdx.x;
if (y < kernelH && x < kernelW) {
int ky = y - kernelY;
if (ky < 0) {
ky += fftH;
}
int kx = x - kernelX;
if (kx < 0) {
kx += fftW;
}
d_Dst[ky * fftW + kx] = LOAD_FLOAT(y * kernelW + x);
}
}
////////////////////////////////////////////////////////////////////////////////
// Prepare data for "pad to border" addressing mode
////////////////////////////////////////////////////////////////////////////////
__global__ void padDataClampToBorder_kernel(float *d_Dst,
float *d_Src,
int fftH,
int fftW,
int dataH,
int dataW,
int kernelH,
int kernelW,
int kernelY,
int kernelX
#if (USE_TEXTURE)
,
cudaTextureObject_t texFloat
#endif
)
{
const int y = blockDim.y * blockIdx.y + threadIdx.y;
const int x = blockDim.x * blockIdx.x + threadIdx.x;
const int borderH = dataH + kernelY;
const int borderW = dataW + kernelX;
if (y < fftH && x < fftW) {
int dy, dx;
if (y < dataH) {
dy = y;
}
if (x < dataW) {
dx = x;
}
if (y >= dataH && y < borderH) {
dy = dataH - 1;
}
if (x >= dataW && x < borderW) {
dx = dataW - 1;
}
if (y >= borderH) {
dy = 0;
}
if (x >= borderW) {
dx = 0;
}
d_Dst[y * fftW + x] = LOAD_FLOAT(dy * dataW + dx);
}
}
////////////////////////////////////////////////////////////////////////////////
// Modulate Fourier image of padded data by Fourier image of padded kernel
// and normalize by FFT size
////////////////////////////////////////////////////////////////////////////////
inline __device__ void mulAndScale(fComplex &a, const fComplex &b, const float &c)
{
fComplex t = {c * (a.x * b.x - a.y * b.y), c * (a.y * b.x + a.x * b.y)};
a = t;
}
__global__ void modulateAndNormalize_kernel(fComplex *d_Dst, fComplex *d_Src, int dataSize, float c)
{
const int i = blockDim.x * blockIdx.x + threadIdx.x;
if (i >= dataSize) {
return;
}
fComplex a = d_Src[i];
fComplex b = d_Dst[i];
mulAndScale(a, b, c);
d_Dst[i] = a;
}
////////////////////////////////////////////////////////////////////////////////
// 2D R2C / C2R post/preprocessing kernels
////////////////////////////////////////////////////////////////////////////////
#if (USE_TEXTURE)
#define LOAD_FCOMPLEX(i) tex1Dfetch<fComplex>(texComplex, i)
#define LOAD_FCOMPLEX_A(i) tex1Dfetch<fComplex>(texComplexA, i)
#define LOAD_FCOMPLEX_B(i) tex1Dfetch<fComplex>(texComplexB, i)
#define SET_FCOMPLEX_BASE
#define SET_FCOMPLEX_BASE_A
#define SET_FCOMPLEX_BASE_B
#else
#define LOAD_FCOMPLEX(i) d_Src[i]
#define LOAD_FCOMPLEX_A(i) d_SrcA[i]
#define LOAD_FCOMPLEX_B(i) d_SrcB[i]
#define SET_FCOMPLEX_BASE
#define SET_FCOMPLEX_BASE_A
#define SET_FCOMPLEX_BASE_B
#endif
inline __device__ void spPostprocessC2C(fComplex &D1, fComplex &D2, const fComplex &twiddle)
{
float A1 = 0.5f * (D1.x + D2.x);
float B1 = 0.5f * (D1.y - D2.y);
float A2 = 0.5f * (D1.y + D2.y);
float B2 = 0.5f * (D1.x - D2.x);
D1.x = A1 + (A2 * twiddle.x + B2 * twiddle.y);
D1.y = (A2 * twiddle.y - B2 * twiddle.x) + B1;
D2.x = A1 - (A2 * twiddle.x + B2 * twiddle.y);
D2.y = (A2 * twiddle.y - B2 * twiddle.x) - B1;
}
// Premultiply by 2 to account for 1.0 / (DZ * DY * DX) normalization
inline __device__ void spPreprocessC2C(fComplex &D1, fComplex &D2, const fComplex &twiddle)
{
float A1 = /* 0.5f * */ (D1.x + D2.x);
float B1 = /* 0.5f * */ (D1.y - D2.y);
float A2 = /* 0.5f * */ (D1.y + D2.y);
float B2 = /* 0.5f * */ (D1.x - D2.x);
D1.x = A1 - (A2 * twiddle.x - B2 * twiddle.y);
D1.y = (B2 * twiddle.x + A2 * twiddle.y) + B1;
D2.x = A1 + (A2 * twiddle.x - B2 * twiddle.y);
D2.y = (B2 * twiddle.x + A2 * twiddle.y) - B1;
}
inline __device__ void getTwiddle(fComplex &twiddle, float phase) { __sincosf(phase, &twiddle.y, &twiddle.x); }
inline __device__ uint mod(uint a, uint DA)
{
//(DA - a) % DA, assuming a <= DA
return a ? (DA - a) : a;
}
static inline uint factorRadix2(uint &log2N, uint n)
{
if (!n) {
log2N = 0;
return 0;
}
else {
for (log2N = 0; n % 2 == 0; n /= 2, log2N++)
;
return n;
}
}
inline __device__ void udivmod(uint &dividend, uint divisor, uint &rem)
{
#if (!POWER_OF_TWO)
rem = dividend % divisor;
dividend /= divisor;
#else
rem = dividend & (divisor - 1);
dividend >>= (__ffs(divisor) - 1);
#endif
}
__global__ void spPostprocess2D_kernel(fComplex *d_Dst,
fComplex *d_Src,
uint DY,
uint DX,
uint threadCount,
uint padding,
float phaseBase
#if (USE_TEXTURE)
,
cudaTextureObject_t texComplex
#endif
)
{
const uint threadId = blockIdx.x * blockDim.x + threadIdx.x;
if (threadId >= threadCount) {
return;
}
uint x, y, i = threadId;
udivmod(i, DX / 2, x);
udivmod(i, DY, y);
// Avoid overwrites in columns DX / 2 by different threads
if ((x == 0) && (y > DY / 2)) {
return;
}
const uint srcOffset = i * DY * DX;
const uint dstOffset = i * DY * (DX + padding);
// Process x = [0 .. DX / 2 - 1] U [DX / 2 + 1 .. DX]
{
const uint loadPos1 = srcOffset + y * DX + x;
const uint loadPos2 = srcOffset + mod(y, DY) * DX + mod(x, DX);
const uint storePos1 = dstOffset + y * (DX + padding) + x;
const uint storePos2 = dstOffset + mod(y, DY) * (DX + padding) + (DX - x);
fComplex D1 = LOAD_FCOMPLEX(loadPos1);
fComplex D2 = LOAD_FCOMPLEX(loadPos2);
fComplex twiddle;
getTwiddle(twiddle, phaseBase * (float)x);
spPostprocessC2C(D1, D2, twiddle);
d_Dst[storePos1] = D1;
d_Dst[storePos2] = D2;
}
// Process x = DX / 2
if (x == 0) {
const uint loadPos1 = srcOffset + y * DX + DX / 2;
const uint loadPos2 = srcOffset + mod(y, DY) * DX + DX / 2;
const uint storePos1 = dstOffset + y * (DX + padding) + DX / 2;
const uint storePos2 = dstOffset + mod(y, DY) * (DX + padding) + DX / 2;
fComplex D1 = LOAD_FCOMPLEX(loadPos1);
fComplex D2 = LOAD_FCOMPLEX(loadPos2);
// twiddle = getTwiddle(phaseBase * (DX / 2)) = exp(dir * j * PI / 2)
fComplex twiddle = {0, (phaseBase > 0) ? 1.0f : -1.0f};
spPostprocessC2C(D1, D2, twiddle);
d_Dst[storePos1] = D1;
d_Dst[storePos2] = D2;
}
}
__global__ void spPreprocess2D_kernel(fComplex *d_Dst,
fComplex *d_Src,
uint DY,
uint DX,
uint threadCount,
uint padding,
float phaseBase
#if (USE_TEXTURE)
,
cudaTextureObject_t texComplex
#endif
)
{
const uint threadId = blockIdx.x * blockDim.x + threadIdx.x;
if (threadId >= threadCount) {
return;
}
uint x, y, i = threadId;
udivmod(i, DX / 2, x);
udivmod(i, DY, y);
// Avoid overwrites in columns 0 and DX / 2 by different threads (lower and
// upper halves)
if ((x == 0) && (y > DY / 2)) {
return;
}
const uint srcOffset = i * DY * (DX + padding);
const uint dstOffset = i * DY * DX;
// Process x = [0 .. DX / 2 - 1] U [DX / 2 + 1 .. DX]
{
const uint loadPos1 = srcOffset + y * (DX + padding) + x;
const uint loadPos2 = srcOffset + mod(y, DY) * (DX + padding) + (DX - x);
const uint storePos1 = dstOffset + y * DX + x;
const uint storePos2 = dstOffset + mod(y, DY) * DX + mod(x, DX);
fComplex D1 = LOAD_FCOMPLEX(loadPos1);
fComplex D2 = LOAD_FCOMPLEX(loadPos2);
fComplex twiddle;
getTwiddle(twiddle, phaseBase * (float)x);
spPreprocessC2C(D1, D2, twiddle);
d_Dst[storePos1] = D1;
d_Dst[storePos2] = D2;
}
// Process x = DX / 2
if (x == 0) {
const uint loadPos1 = srcOffset + y * (DX + padding) + DX / 2;
const uint loadPos2 = srcOffset + mod(y, DY) * (DX + padding) + DX / 2;
const uint storePos1 = dstOffset + y * DX + DX / 2;
const uint storePos2 = dstOffset + mod(y, DY) * DX + DX / 2;
fComplex D1 = LOAD_FCOMPLEX(loadPos1);
fComplex D2 = LOAD_FCOMPLEX(loadPos2);
// twiddle = getTwiddle(phaseBase * (DX / 2)) = exp(-dir * j * PI / 2)
fComplex twiddle = {0, (phaseBase > 0) ? 1.0f : -1.0f};
spPreprocessC2C(D1, D2, twiddle);
d_Dst[storePos1] = D1;
d_Dst[storePos2] = D2;
}
}
////////////////////////////////////////////////////////////////////////////////
// Combined spPostprocess2D + modulateAndNormalize + spPreprocess2D
////////////////////////////////////////////////////////////////////////////////
__global__ void spProcess2D_kernel(fComplex *d_Dst,
fComplex *d_SrcA,
fComplex *d_SrcB,
uint DY,
uint DX,
uint threadCount,
float phaseBase,
float c
#if (USE_TEXTURE)
,
cudaTextureObject_t texComplexA,
cudaTextureObject_t texComplexB
#endif
)
{
const uint threadId = blockIdx.x * blockDim.x + threadIdx.x;
if (threadId >= threadCount) {
return;
}
uint x, y, i = threadId;
udivmod(i, DX, x);
udivmod(i, DY / 2, y);
const uint offset = i * DY * DX;
// Avoid overwrites in rows 0 and DY / 2 by different threads (left and right
// halves) Otherwise correctness for in-place transformations is affected
if ((y == 0) && (x > DX / 2)) {
return;
}
fComplex twiddle;
// Process y = [0 .. DY / 2 - 1] U [DY - (DY / 2) + 1 .. DY - 1]
{
const uint pos1 = offset + y * DX + x;
const uint pos2 = offset + mod(y, DY) * DX + mod(x, DX);
fComplex D1 = LOAD_FCOMPLEX_A(pos1);
fComplex D2 = LOAD_FCOMPLEX_A(pos2);
fComplex K1 = LOAD_FCOMPLEX_B(pos1);
fComplex K2 = LOAD_FCOMPLEX_B(pos2);
getTwiddle(twiddle, phaseBase * (float)x);
spPostprocessC2C(D1, D2, twiddle);
spPostprocessC2C(K1, K2, twiddle);
mulAndScale(D1, K1, c);
mulAndScale(D2, K2, c);
spPreprocessC2C(D1, D2, twiddle);
d_Dst[pos1] = D1;
d_Dst[pos2] = D2;
}
if (y == 0) {
const uint pos1 = offset + (DY / 2) * DX + x;
const uint pos2 = offset + (DY / 2) * DX + mod(x, DX);
fComplex D1 = LOAD_FCOMPLEX_A(pos1);
fComplex D2 = LOAD_FCOMPLEX_A(pos2);
fComplex K1 = LOAD_FCOMPLEX_B(pos1);
fComplex K2 = LOAD_FCOMPLEX_B(pos2);
spPostprocessC2C(D1, D2, twiddle);
spPostprocessC2C(K1, K2, twiddle);
mulAndScale(D1, K1, c);
mulAndScale(D2, K2, c);
spPreprocessC2C(D1, D2, twiddle);
d_Dst[pos1] = D1;
d_Dst[pos2] = D2;
}
}

90
cpp/5_Domain_Specific/convolutionFFT2D/convolutionFFT2D_common.h Normal file
View File

@@ -0,0 +1,90 @@
/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of NVIDIA CORPORATION nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
* OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef CONVOLUTIONFFT2D_COMMON_H
#define CONVOLUTIONFFT2D_COMMON_H
typedef unsigned int uint;
#ifdef __CUDACC__
typedef float2 fComplex;
#else
typedef struct
{
float x;
float y;
} fComplex;
#endif
////////////////////////////////////////////////////////////////////////////////
// Helper functions
////////////////////////////////////////////////////////////////////////////////
// Round a / b to nearest higher integer value
inline int iDivUp(int a, int b) { return (a % b != 0) ? (a / b + 1) : (a / b); }
// Align a to nearest higher multiple of b
inline int iAlignUp(int a, int b) { return (a % b != 0) ? (a - a % b + b) : a; }
extern "C" void convolutionClampToBorderCPU(float *h_Result,
float *h_Data,
float *h_Kernel,
int dataH,
int dataW,
int kernelH,
int kernelW,
int kernelY,
int kernelX);
extern "C" void padKernel(float *d_PaddedKernel,
float *d_Kernel,
int fftH,
int fftW,
int kernelH,
int kernelW,
int kernelY,
int kernelX);
extern "C" void padDataClampToBorder(float *d_PaddedData,
float *d_Data,
int fftH,
int fftW,
int dataH,
int dataW,
int kernelH,
int kernelW,
int kernelY,
int kernelX);
extern "C" void modulateAndNormalize(fComplex *d_Dst, fComplex *d_Src, int fftH, int fftW, int padding);
extern "C" void spPostprocess2D(void *d_Dst, void *d_Src, uint DY, uint DX, uint padding, int dir);
extern "C" void spPreprocess2D(void *d_Dst, void *d_Src, uint DY, uint DX, uint padding, int dir);
extern "C" void spProcess2D(void *d_Data, void *d_Data0, void *d_Kernel0, uint DY, uint DX, int dir);
#endif // CONVOLUTIONFFT2D_COMMON_H

71
cpp/5_Domain_Specific/convolutionFFT2D/convolutionFFT2D_gold.cpp Normal file
View File

@@ -0,0 +1,71 @@
/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of NVIDIA CORPORATION nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
* OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include <assert.h>
#include "convolutionFFT2D_common.h"
////////////////////////////////////////////////////////////////////////////////
// Reference straightforward CPU convolution
////////////////////////////////////////////////////////////////////////////////
extern "C" void convolutionClampToBorderCPU(float *h_Result,
float *h_Data,
float *h_Kernel,
int dataH,
int dataW,
int kernelH,
int kernelW,
int kernelY,
int kernelX)
{
for (int y = 0; y < dataH; y++)
for (int x = 0; x < dataW; x++) {
double sum = 0;
for (int ky = -(kernelH - kernelY - 1); ky <= kernelY; ky++)
for (int kx = -(kernelW - kernelX - 1); kx <= kernelX; kx++) {
int dy = y + ky;
int dx = x + kx;
if (dy < 0)
dy = 0;
if (dx < 0)
dx = 0;
if (dy >= dataH)
dy = dataH - 1;
if (dx >= dataW)
dx = dataW - 1;
sum += h_Data[dy * dataW + dx] * h_Kernel[(kernelY - ky) * kernelW + (kernelX - kx)];
}
h_Result[y * dataW + x] = (float)sum;
}
}

530
cpp/5_Domain_Specific/convolutionFFT2D/main.cpp Normal file
View File

@@ -0,0 +1,530 @@
/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of NVIDIA CORPORATION nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
* OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
/*
* This sample demonstrates how 2D convolutions
* with very large kernel sizes
* can be efficiently implemented
* using FFT transformations.
*/
#include <assert.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
// Include CUDA runtime and CUFFT
#include <cuda_runtime.h>
#include <cufft.h>
// Helper functions for CUDA
#include <helper_cuda.h>
#include <helper_functions.h>
#include "convolutionFFT2D_common.h"
////////////////////////////////////////////////////////////////////////////////
// Helper functions
////////////////////////////////////////////////////////////////////////////////
int snapTransformSize(int dataSize)
{
int hiBit;
unsigned int lowPOT, hiPOT;
dataSize = iAlignUp(dataSize, 16);
for (hiBit = 31; hiBit >= 0; hiBit--)
if (dataSize & (1U << hiBit)) {
break;
}
lowPOT = 1U << hiBit;
if (lowPOT == (unsigned int)dataSize) {
return dataSize;
}
hiPOT = 1U << (hiBit + 1);
if (hiPOT <= 1024) {
return hiPOT;
}
else {
return iAlignUp(dataSize, 512);
}
}
float getRand(void) { return (float)(rand() % 16); }
bool test0(void)
{
float *h_Data, *h_Kernel, *h_ResultCPU, *h_ResultGPU;
float *d_Data, *d_PaddedData, *d_Kernel, *d_PaddedKernel;
fComplex *d_DataSpectrum, *d_KernelSpectrum;
cufftHandle fftPlanFwd, fftPlanInv;
bool bRetVal;
StopWatchInterface *hTimer = NULL;
sdkCreateTimer(&hTimer);
printf("Testing built-in R2C / C2R FFT-based convolution\n");
const int kernelH = 7;
const int kernelW = 6;
const int kernelY = 3;
const int kernelX = 4;
const int dataH = 2000;
const int dataW = 2000;
const int fftH = snapTransformSize(dataH + kernelH - 1);
const int fftW = snapTransformSize(dataW + kernelW - 1);
printf("...allocating memory\n");
h_Data = (float *)malloc(dataH * dataW * sizeof(float));
h_Kernel = (float *)malloc(kernelH * kernelW * sizeof(float));
h_ResultCPU = (float *)malloc(dataH * dataW * sizeof(float));
h_ResultGPU = (float *)malloc(fftH * fftW * sizeof(float));
checkCudaErrors(cudaMalloc((void **)&d_Data, dataH * dataW * sizeof(float)));
checkCudaErrors(cudaMalloc((void **)&d_Kernel, kernelH * kernelW * sizeof(float)));
checkCudaErrors(cudaMalloc((void **)&d_PaddedData, fftH * fftW * sizeof(float)));
checkCudaErrors(cudaMalloc((void **)&d_PaddedKernel, fftH * fftW * sizeof(float)));
checkCudaErrors(cudaMalloc((void **)&d_DataSpectrum, fftH * (fftW / 2 + 1) * sizeof(fComplex)));
checkCudaErrors(cudaMalloc((void **)&d_KernelSpectrum, fftH * (fftW / 2 + 1) * sizeof(fComplex)));
checkCudaErrors(cudaMemset(d_KernelSpectrum, 0, fftH * (fftW / 2 + 1) * sizeof(fComplex)));
printf("...generating random input data\n");
srand(2010);
for (int i = 0; i < dataH * dataW; i++) {
h_Data[i] = getRand();
}
for (int i = 0; i < kernelH * kernelW; i++) {
h_Kernel[i] = getRand();
}
printf("...creating R2C & C2R FFT plans for %i x %i\n", fftH, fftW);
checkCudaErrors(cufftPlan2d(&fftPlanFwd, fftH, fftW, CUFFT_R2C));
checkCudaErrors(cufftPlan2d(&fftPlanInv, fftH, fftW, CUFFT_C2R));
printf("...uploading to GPU and padding convolution kernel and input data\n");
checkCudaErrors(cudaMemcpy(d_Kernel, h_Kernel, kernelH * kernelW * sizeof(float), cudaMemcpyHostToDevice));
checkCudaErrors(cudaMemcpy(d_Data, h_Data, dataH * dataW * sizeof(float), cudaMemcpyHostToDevice));
checkCudaErrors(cudaMemset(d_PaddedKernel, 0, fftH * fftW * sizeof(float)));
checkCudaErrors(cudaMemset(d_PaddedData, 0, fftH * fftW * sizeof(float)));
padKernel(d_PaddedKernel, d_Kernel, fftH, fftW, kernelH, kernelW, kernelY, kernelX);
padDataClampToBorder(d_PaddedData, d_Data, fftH, fftW, dataH, dataW, kernelH, kernelW, kernelY, kernelX);
// Not including kernel transformation into time measurement,
// since convolution kernel is not changed very frequently
printf("...transforming convolution kernel\n");
checkCudaErrors(cufftExecR2C(fftPlanFwd, (cufftReal *)d_PaddedKernel, (cufftComplex *)d_KernelSpectrum));
printf("...running GPU FFT convolution: ");
checkCudaErrors(cudaDeviceSynchronize());
sdkResetTimer(&hTimer);
sdkStartTimer(&hTimer);
checkCudaErrors(cufftExecR2C(fftPlanFwd, (cufftReal *)d_PaddedData, (cufftComplex *)d_DataSpectrum));
modulateAndNormalize(d_DataSpectrum, d_KernelSpectrum, fftH, fftW, 1);
checkCudaErrors(cufftExecC2R(fftPlanInv, (cufftComplex *)d_DataSpectrum, (cufftReal *)d_PaddedData));
checkCudaErrors(cudaDeviceSynchronize());
sdkStopTimer(&hTimer);
double gpuTime = sdkGetTimerValue(&hTimer);
printf("%f MPix/s (%f ms)\n", (double)dataH * (double)dataW * 1e-6 / (gpuTime * 0.001), gpuTime);
printf("...reading back GPU convolution results\n");
checkCudaErrors(cudaMemcpy(h_ResultGPU, d_PaddedData, fftH * fftW * sizeof(float), cudaMemcpyDeviceToHost));
printf("...running reference CPU convolution\n");
convolutionClampToBorderCPU(h_ResultCPU, h_Data, h_Kernel, dataH, dataW, kernelH, kernelW, kernelY, kernelX);
printf("...comparing the results: ");
double sum_delta2 = 0;
double sum_ref2 = 0;
double max_delta_ref = 0;
for (int y = 0; y < dataH; y++)
for (int x = 0; x < dataW; x++) {
double rCPU = (double)h_ResultCPU[y * dataW + x];
double rGPU = (double)h_ResultGPU[y * fftW + x];
double delta = (rCPU - rGPU) * (rCPU - rGPU);
double ref = rCPU * rCPU + rCPU * rCPU;
if ((delta / ref) > max_delta_ref) {
max_delta_ref = delta / ref;
}
sum_delta2 += delta;
sum_ref2 += ref;
}
double L2norm = sqrt(sum_delta2 / sum_ref2);
printf("rel L2 = %E (max delta = %E)\n", L2norm, sqrt(max_delta_ref));
bRetVal = (L2norm < 1e-6) ? true : false;
printf(bRetVal ? "L2norm Error OK\n" : "L2norm Error too high!\n");
printf("...shutting down\n");
sdkDeleteTimer(&hTimer);
checkCudaErrors(cufftDestroy(fftPlanInv));
checkCudaErrors(cufftDestroy(fftPlanFwd));
checkCudaErrors(cudaFree(d_DataSpectrum));
checkCudaErrors(cudaFree(d_KernelSpectrum));
checkCudaErrors(cudaFree(d_PaddedData));
checkCudaErrors(cudaFree(d_PaddedKernel));
checkCudaErrors(cudaFree(d_Data));
checkCudaErrors(cudaFree(d_Kernel));
free(h_ResultGPU);
free(h_ResultCPU);
free(h_Data);
free(h_Kernel);
return bRetVal;
}
bool test1(void)
{
float *h_Data, *h_Kernel, *h_ResultCPU, *h_ResultGPU;
float *d_Data, *d_Kernel, *d_PaddedData, *d_PaddedKernel;
fComplex *d_DataSpectrum0, *d_KernelSpectrum0, *d_DataSpectrum, *d_KernelSpectrum;
cufftHandle fftPlan;
bool bRetVal;
StopWatchInterface *hTimer = NULL;
sdkCreateTimer(&hTimer);
printf("Testing custom R2C / C2R FFT-based convolution\n");
const uint fftPadding = 16;
const int kernelH = 7;
const int kernelW = 6;
const int kernelY = 3;
const int kernelX = 4;
const int dataH = 2000;
const int dataW = 2000;
const int fftH = snapTransformSize(dataH + kernelH - 1);
const int fftW = snapTransformSize(dataW + kernelW - 1);
printf("...allocating memory\n");
h_Data = (float *)malloc(dataH * dataW * sizeof(float));
h_Kernel = (float *)malloc(kernelH * kernelW * sizeof(float));
h_ResultCPU = (float *)malloc(dataH * dataW * sizeof(float));
h_ResultGPU = (float *)malloc(fftH * fftW * sizeof(float));
checkCudaErrors(cudaMalloc((void **)&d_Data, dataH * dataW * sizeof(float)));
checkCudaErrors(cudaMalloc((void **)&d_Kernel, kernelH * kernelW * sizeof(float)));
checkCudaErrors(cudaMalloc((void **)&d_PaddedData, fftH * fftW * sizeof(float)));
checkCudaErrors(cudaMalloc((void **)&d_PaddedKernel, fftH * fftW * sizeof(float)));
checkCudaErrors(cudaMalloc((void **)&d_DataSpectrum0, fftH * (fftW / 2) * sizeof(fComplex)));
checkCudaErrors(cudaMalloc((void **)&d_KernelSpectrum0, fftH * (fftW / 2) * sizeof(fComplex)));
checkCudaErrors(cudaMalloc((void **)&d_DataSpectrum, fftH * (fftW / 2 + fftPadding) * sizeof(fComplex)));
checkCudaErrors(cudaMalloc((void **)&d_KernelSpectrum, fftH * (fftW / 2 + fftPadding) * sizeof(fComplex)));
printf("...generating random input data\n");
srand(2010);
for (int i = 0; i < dataH * dataW; i++) {
h_Data[i] = getRand();
}
for (int i = 0; i < kernelH * kernelW; i++) {
h_Kernel[i] = getRand();
}
printf("...creating C2C FFT plan for %i x %i\n", fftH, fftW / 2);
checkCudaErrors(cufftPlan2d(&fftPlan, fftH, fftW / 2, CUFFT_C2C));
printf("...uploading to GPU and padding convolution kernel and input data\n");
checkCudaErrors(cudaMemcpy(d_Data, h_Data, dataH * dataW * sizeof(float), cudaMemcpyHostToDevice));
checkCudaErrors(cudaMemcpy(d_Kernel, h_Kernel, kernelH * kernelW * sizeof(float), cudaMemcpyHostToDevice));
checkCudaErrors(cudaMemset(d_PaddedData, 0, fftH * fftW * sizeof(float)));
checkCudaErrors(cudaMemset(d_PaddedKernel, 0, fftH * fftW * sizeof(float)));
padDataClampToBorder(d_PaddedData, d_Data, fftH, fftW, dataH, dataW, kernelH, kernelW, kernelY, kernelX);
padKernel(d_PaddedKernel, d_Kernel, fftH, fftW, kernelH, kernelW, kernelY, kernelX);
// CUFFT_INVERSE works just as well...
const int FFT_DIR = CUFFT_FORWARD;
// Not including kernel transformation into time measurement,
// since convolution kernel is not changed very frequently
printf("...transforming convolution kernel\n");
checkCudaErrors(cufftExecC2C(fftPlan, (cufftComplex *)d_PaddedKernel, (cufftComplex *)d_KernelSpectrum0, FFT_DIR));
spPostprocess2D(d_KernelSpectrum, d_KernelSpectrum0, fftH, fftW / 2, fftPadding, FFT_DIR);
printf("...running GPU FFT convolution: ");
checkCudaErrors(cudaDeviceSynchronize());
sdkResetTimer(&hTimer);
sdkStartTimer(&hTimer);
checkCudaErrors(cufftExecC2C(fftPlan, (cufftComplex *)d_PaddedData, (cufftComplex *)d_DataSpectrum0, FFT_DIR));
spPostprocess2D(d_DataSpectrum, d_DataSpectrum0, fftH, fftW / 2, fftPadding, FFT_DIR);
modulateAndNormalize(d_DataSpectrum, d_KernelSpectrum, fftH, fftW, fftPadding);
spPreprocess2D(d_DataSpectrum0, d_DataSpectrum, fftH, fftW / 2, fftPadding, -FFT_DIR);
checkCudaErrors(cufftExecC2C(fftPlan, (cufftComplex *)d_DataSpectrum0, (cufftComplex *)d_PaddedData, -FFT_DIR));
checkCudaErrors(cudaDeviceSynchronize());
sdkStopTimer(&hTimer);
double gpuTime = sdkGetTimerValue(&hTimer);
printf("%f MPix/s (%f ms)\n", (double)dataH * (double)dataW * 1e-6 / (gpuTime * 0.001), gpuTime);
printf("...reading back GPU FFT results\n");
checkCudaErrors(cudaMemcpy(h_ResultGPU, d_PaddedData, fftH * fftW * sizeof(float), cudaMemcpyDeviceToHost));
printf("...running reference CPU convolution\n");
convolutionClampToBorderCPU(h_ResultCPU, h_Data, h_Kernel, dataH, dataW, kernelH, kernelW, kernelY, kernelX);
printf("...comparing the results: ");
double sum_delta2 = 0;
double sum_ref2 = 0;
double max_delta_ref = 0;
for (int y = 0; y < dataH; y++)
for (int x = 0; x < dataW; x++) {
double rCPU = (double)h_ResultCPU[y * dataW + x];
double rGPU = (double)h_ResultGPU[y * fftW + x];
double delta = (rCPU - rGPU) * (rCPU - rGPU);
double ref = rCPU * rCPU + rCPU * rCPU;
if ((delta / ref) > max_delta_ref) {
max_delta_ref = delta / ref;
}
sum_delta2 += delta;
sum_ref2 += ref;
}
double L2norm = sqrt(sum_delta2 / sum_ref2);
printf("rel L2 = %E (max delta = %E)\n", L2norm, sqrt(max_delta_ref));
bRetVal = (L2norm < 1e-6) ? true : false;
printf(bRetVal ? "L2norm Error OK\n" : "L2norm Error too high!\n");
printf("...shutting down\n");
sdkDeleteTimer(&hTimer);
checkCudaErrors(cufftDestroy(fftPlan));
checkCudaErrors(cudaFree(d_KernelSpectrum));
checkCudaErrors(cudaFree(d_DataSpectrum));
checkCudaErrors(cudaFree(d_KernelSpectrum0));
checkCudaErrors(cudaFree(d_DataSpectrum0));
checkCudaErrors(cudaFree(d_PaddedKernel));
checkCudaErrors(cudaFree(d_PaddedData));
checkCudaErrors(cudaFree(d_Kernel));
checkCudaErrors(cudaFree(d_Data));
free(h_ResultGPU);
free(h_ResultCPU);
free(h_Kernel);
free(h_Data);
return bRetVal;
}
bool test2(void)
{
float *h_Data, *h_Kernel, *h_ResultCPU, *h_ResultGPU;
float *d_Data, *d_Kernel, *d_PaddedData, *d_PaddedKernel;
fComplex *d_DataSpectrum0, *d_KernelSpectrum0;
cufftHandle fftPlan;
bool bRetVal;
StopWatchInterface *hTimer = NULL;
sdkCreateTimer(&hTimer);
printf("Testing updated custom R2C / C2R FFT-based convolution\n");
const int kernelH = 7;
const int kernelW = 6;
const int kernelY = 3;
const int kernelX = 4;
const int dataH = 2000;
const int dataW = 2000;
const int fftH = snapTransformSize(dataH + kernelH - 1);
const int fftW = snapTransformSize(dataW + kernelW - 1);
printf("...allocating memory\n");
h_Data = (float *)malloc(dataH * dataW * sizeof(float));
h_Kernel = (float *)malloc(kernelH * kernelW * sizeof(float));
h_ResultCPU = (float *)malloc(dataH * dataW * sizeof(float));
h_ResultGPU = (float *)malloc(fftH * fftW * sizeof(float));
checkCudaErrors(cudaMalloc((void **)&d_Data, dataH * dataW * sizeof(float)));
checkCudaErrors(cudaMalloc((void **)&d_Kernel, kernelH * kernelW * sizeof(float)));
checkCudaErrors(cudaMalloc((void **)&d_PaddedData, fftH * fftW * sizeof(float)));
checkCudaErrors(cudaMalloc((void **)&d_PaddedKernel, fftH * fftW * sizeof(float)));
checkCudaErrors(cudaMalloc((void **)&d_DataSpectrum0, fftH * (fftW / 2) * sizeof(fComplex)));
checkCudaErrors(cudaMalloc((void **)&d_KernelSpectrum0, fftH * (fftW / 2) * sizeof(fComplex)));
printf("...generating random input data\n");
srand(2010);
for (int i = 0; i < dataH * dataW; i++) {
h_Data[i] = getRand();
}
for (int i = 0; i < kernelH * kernelW; i++) {
h_Kernel[i] = getRand();
}
printf("...creating C2C FFT plan for %i x %i\n", fftH, fftW / 2);
checkCudaErrors(cufftPlan2d(&fftPlan, fftH, fftW / 2, CUFFT_C2C));
printf("...uploading to GPU and padding convolution kernel and input data\n");
checkCudaErrors(cudaMemcpy(d_Data, h_Data, dataH * dataW * sizeof(float), cudaMemcpyHostToDevice));
checkCudaErrors(cudaMemcpy(d_Kernel, h_Kernel, kernelH * kernelW * sizeof(float), cudaMemcpyHostToDevice));
checkCudaErrors(cudaMemset(d_PaddedData, 0, fftH * fftW * sizeof(float)));
checkCudaErrors(cudaMemset(d_PaddedKernel, 0, fftH * fftW * sizeof(float)));
padDataClampToBorder(d_PaddedData, d_Data, fftH, fftW, dataH, dataW, kernelH, kernelW, kernelY, kernelX);
padKernel(d_PaddedKernel, d_Kernel, fftH, fftW, kernelH, kernelW, kernelY, kernelX);
// CUFFT_INVERSE works just as well...
const int FFT_DIR = CUFFT_FORWARD;
// Not including kernel transformation into time measurement,
// since convolution kernel is not changed very frequently
printf("...transforming convolution kernel\n");
checkCudaErrors(cufftExecC2C(fftPlan, (cufftComplex *)d_PaddedKernel, (cufftComplex *)d_KernelSpectrum0, FFT_DIR));
printf("...running GPU FFT convolution: ");
checkCudaErrors(cudaDeviceSynchronize());
sdkResetTimer(&hTimer);
sdkStartTimer(&hTimer);
checkCudaErrors(cufftExecC2C(fftPlan, (cufftComplex *)d_PaddedData, (cufftComplex *)d_DataSpectrum0, FFT_DIR));
spProcess2D(d_DataSpectrum0, d_DataSpectrum0, d_KernelSpectrum0, fftH, fftW / 2, FFT_DIR);
checkCudaErrors(cufftExecC2C(fftPlan, (cufftComplex *)d_DataSpectrum0, (cufftComplex *)d_PaddedData, -FFT_DIR));
checkCudaErrors(cudaDeviceSynchronize());
sdkStopTimer(&hTimer);
double gpuTime = sdkGetTimerValue(&hTimer);
printf("%f MPix/s (%f ms)\n", (double)dataH * (double)dataW * 1e-6 / (gpuTime * 0.001), gpuTime);
printf("...reading back GPU FFT results\n");
checkCudaErrors(cudaMemcpy(h_ResultGPU, d_PaddedData, fftH * fftW * sizeof(float), cudaMemcpyDeviceToHost));
printf("...running reference CPU convolution\n");
convolutionClampToBorderCPU(h_ResultCPU, h_Data, h_Kernel, dataH, dataW, kernelH, kernelW, kernelY, kernelX);
printf("...comparing the results: ");
double sum_delta2 = 0;
double sum_ref2 = 0;
double max_delta_ref = 0;
for (int y = 0; y < dataH; y++) {
for (int x = 0; x < dataW; x++) {
double rCPU = (double)h_ResultCPU[y * dataW + x];
double rGPU = (double)h_ResultGPU[y * fftW + x];
double delta = (rCPU - rGPU) * (rCPU - rGPU);
double ref = rCPU * rCPU + rCPU * rCPU;
if ((delta / ref) > max_delta_ref) {
max_delta_ref = delta / ref;
}
sum_delta2 += delta;
sum_ref2 += ref;
}
}
double L2norm = sqrt(sum_delta2 / sum_ref2);
printf("rel L2 = %E (max delta = %E)\n", L2norm, sqrt(max_delta_ref));
bRetVal = (L2norm < 1e-6) ? true : false;
printf(bRetVal ? "L2norm Error OK\n" : "L2norm Error too high!\n");
printf("...shutting down\n");
sdkDeleteTimer(&hTimer);
checkCudaErrors(cufftDestroy(fftPlan));
checkCudaErrors(cudaFree(d_KernelSpectrum0));
checkCudaErrors(cudaFree(d_DataSpectrum0));
checkCudaErrors(cudaFree(d_PaddedKernel));
checkCudaErrors(cudaFree(d_PaddedData));
checkCudaErrors(cudaFree(d_Kernel));
checkCudaErrors(cudaFree(d_Data));
free(h_ResultGPU);
free(h_ResultCPU);
free(h_Kernel);
free(h_Data);
return bRetVal;
}
int main(int argc, char **argv)
{
printf("[%s] - Starting...\n", argv[0]);
// Use command-line specified CUDA device, otherwise use device with highest
// Gflops/s
findCudaDevice(argc, (const char **)argv);
int nFailures = 0;
if (!test0()) {
nFailures++;
}
if (!test1()) {
nFailures++;
}
if (!test2()) {
nFailures++;
}
printf("Test Summary: %d errors\n", nFailures);
if (nFailures > 0) {
printf("Test failed!\n");
exit(EXIT_FAILURE);
}
printf("Test passed\n");
exit(EXIT_SUCCESS);
}