NASP/cuda-samples
85
0
Fork 0
mirror of https://github.com/NVIDIA/cuda-samples.git synced 2026-10-25 22:28:17 +08:00
Code Issues Packages Projects Releases Wiki Activity

Apply consistent code formatting across the repo. Add clang-format and pre-commit hooks.

Browse Source
This commit is contained in:
Rob Armstrong
2025-03-27 10:30:07 -07:00
parent 2cd58fbc9a
commit ceab6e8bcc
782 changed files with 107230 additions and 106548 deletions
Download Patch File Download Diff File Expand all files Collapse all files

387
Samples/5_Domain_Specific/convolutionFFT2D/convolutionFFT2D.cu
View File

@@ -29,98 +29,124 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
// Other helpers
#include <helper_cuda.h>
#include "convolutionFFT2D_common.h"
// Project includes
#include "convolutionFFT2D.cuh"
#include "convolutionFFT2D_common.h"
////////////////////////////////////////////////////////////////////////////////
/// Position convolution kernel center at (0, 0) in the image
////////////////////////////////////////////////////////////////////////////////
extern "C" void padKernel(float *d_Dst, float *d_Src, int fftH, int fftW,
int kernelH, int kernelW, int kernelY, int kernelX) {
assert(d_Src != d_Dst);
dim3 threads(32, 8);
dim3 grid(iDivUp(kernelW, threads.x), iDivUp(kernelH, threads.y));
extern "C" void
padKernel(float *d_Dst, float *d_Src, int fftH, int fftW, int kernelH, int kernelW, int kernelY, int kernelX)
{
assert(d_Src != d_Dst);
dim3 threads(32, 8);
dim3 grid(iDivUp(kernelW, threads.x), iDivUp(kernelH, threads.y));
SET_FLOAT_BASE;
SET_FLOAT_BASE;
#if (USE_TEXTURE)
cudaTextureObject_t texFloat;
cudaResourceDesc texRes;
memset(&texRes, 0, sizeof(cudaResourceDesc));
cudaTextureObject_t texFloat;
cudaResourceDesc texRes;
memset(&texRes, 0, sizeof(cudaResourceDesc));
texRes.resType = cudaResourceTypeLinear;
texRes.res.linear.devPtr = d_Src;
texRes.res.linear.sizeInBytes = sizeof(float) * kernelH * kernelW;
texRes.res.linear.desc = cudaCreateChannelDesc<float>();
texRes.resType = cudaResourceTypeLinear;
texRes.res.linear.devPtr = d_Src;
texRes.res.linear.sizeInBytes = sizeof(float) * kernelH * kernelW;
texRes.res.linear.desc = cudaCreateChannelDesc<float>();
cudaTextureDesc texDescr;
memset(&texDescr, 0, sizeof(cudaTextureDesc));
cudaTextureDesc texDescr;
memset(&texDescr, 0, sizeof(cudaTextureDesc));
texDescr.normalizedCoords = false;
texDescr.filterMode = cudaFilterModeLinear;
texDescr.addressMode[0] = cudaAddressModeWrap;
texDescr.readMode = cudaReadModeElementType;
texDescr.normalizedCoords = false;
texDescr.filterMode = cudaFilterModeLinear;
texDescr.addressMode[0] = cudaAddressModeWrap;
texDescr.readMode = cudaReadModeElementType;
checkCudaErrors(cudaCreateTextureObject(&texFloat, &texRes, &texDescr, NULL));
checkCudaErrors(cudaCreateTextureObject(&texFloat, &texRes, &texDescr, NULL));
#endif
padKernel_kernel<<<grid, threads>>>(d_Dst, d_Src, fftH, fftW, kernelH,
kernelW, kernelY, kernelX
padKernel_kernel<<<grid, threads>>>(d_Dst,
d_Src,
fftH,
fftW,
kernelH,
kernelW,
kernelY,
kernelX
#if (USE_TEXTURE)
,
texFloat
,
texFloat
#endif
);
getLastCudaError("padKernel_kernel<<<>>> execution failed\n");
);
getLastCudaError("padKernel_kernel<<<>>> execution failed\n");
#if (USE_TEXTURE)
checkCudaErrors(cudaDestroyTextureObject(texFloat));
checkCudaErrors(cudaDestroyTextureObject(texFloat));
#endif
}
////////////////////////////////////////////////////////////////////////////////
// Prepare data for "pad to border" addressing mode
////////////////////////////////////////////////////////////////////////////////
extern "C" void padDataClampToBorder(float *d_Dst, float *d_Src, int fftH,
int fftW, int dataH, int dataW,
int kernelW, int kernelH, int kernelY,
int kernelX) {
assert(d_Src != d_Dst);
dim3 threads(32, 8);
dim3 grid(iDivUp(fftW, threads.x), iDivUp(fftH, threads.y));
extern "C" void padDataClampToBorder(float *d_Dst,
float *d_Src,
int fftH,
int fftW,
int dataH,
int dataW,
int kernelW,
int kernelH,
int kernelY,
int kernelX)
{
assert(d_Src != d_Dst);
dim3 threads(32, 8);
dim3 grid(iDivUp(fftW, threads.x), iDivUp(fftH, threads.y));
#if (USE_TEXTURE)
cudaTextureObject_t texFloat;
cudaResourceDesc texRes;
memset(&texRes, 0, sizeof(cudaResourceDesc));
cudaTextureObject_t texFloat;
cudaResourceDesc texRes;
memset(&texRes, 0, sizeof(cudaResourceDesc));
texRes.resType = cudaResourceTypeLinear;
texRes.res.linear.devPtr = d_Src;
texRes.res.linear.sizeInBytes = sizeof(float) * dataH * dataW;
texRes.res.linear.desc = cudaCreateChannelDesc<float>();
texRes.resType = cudaResourceTypeLinear;
texRes.res.linear.devPtr = d_Src;
texRes.res.linear.sizeInBytes = sizeof(float) * dataH * dataW;
texRes.res.linear.desc = cudaCreateChannelDesc<float>();
cudaTextureDesc texDescr;
memset(&texDescr, 0, sizeof(cudaTextureDesc));
cudaTextureDesc texDescr;
memset(&texDescr, 0, sizeof(cudaTextureDesc));
texDescr.normalizedCoords = false;
texDescr.filterMode = cudaFilterModeLinear;
texDescr.addressMode[0] = cudaAddressModeWrap;
texDescr.readMode = cudaReadModeElementType;
texDescr.normalizedCoords = false;
texDescr.filterMode = cudaFilterModeLinear;
texDescr.addressMode[0] = cudaAddressModeWrap;
texDescr.readMode = cudaReadModeElementType;
checkCudaErrors(cudaCreateTextureObject(&texFloat, &texRes, &texDescr, NULL));
checkCudaErrors(cudaCreateTextureObject(&texFloat, &texRes, &texDescr, NULL));
#endif
padDataClampToBorder_kernel<<<grid, threads>>>(
d_Dst, d_Src, fftH, fftW, dataH, dataW, kernelH, kernelW, kernelY, kernelX
padDataClampToBorder_kernel<<<grid, threads>>>(d_Dst,
d_Src,
fftH,
fftW,
dataH,
dataW,
kernelH,
kernelW,
kernelY,
kernelX
#if (USE_TEXTURE)
,
texFloat
,
texFloat
#endif
);
getLastCudaError("padDataClampToBorder_kernel<<<>>> execution failed\n");
);
getLastCudaError("padDataClampToBorder_kernel<<<>>> execution failed\n");
#if (USE_TEXTURE)
checkCudaErrors(cudaDestroyTextureObject(texFloat));
checkCudaErrors(cudaDestroyTextureObject(texFloat));
#endif
}
@@ -128,192 +154,201 @@ extern "C" void padDataClampToBorder(float *d_Dst, float *d_Src, int fftH,
// Modulate Fourier image of padded data by Fourier image of padded kernel
// and normalize by FFT size
////////////////////////////////////////////////////////////////////////////////
extern "C" void modulateAndNormalize(fComplex *d_Dst, fComplex *d_Src, int fftH,
int fftW, int padding) {
assert(fftW % 2 == 0);
const int dataSize = fftH * (fftW / 2 + padding);
extern "C" void modulateAndNormalize(fComplex *d_Dst, fComplex *d_Src, int fftH, int fftW, int padding)
{
assert(fftW % 2 == 0);
const int dataSize = fftH * (fftW / 2 + padding);
modulateAndNormalize_kernel<<<iDivUp(dataSize, 256), 256>>>(
d_Dst, d_Src, dataSize, 1.0f / (float)(fftW * fftH));
getLastCudaError("modulateAndNormalize() execution failed\n");
modulateAndNormalize_kernel<<<iDivUp(dataSize, 256), 256>>>(d_Dst, d_Src, dataSize, 1.0f / (float)(fftW * fftH));
getLastCudaError("modulateAndNormalize() execution failed\n");
}
////////////////////////////////////////////////////////////////////////////////
// 2D R2C / C2R post/preprocessing kernels
////////////////////////////////////////////////////////////////////////////////
static const double PI = 3.1415926535897932384626433832795;
static const uint BLOCKDIM = 256;
static const double PI = 3.1415926535897932384626433832795;
static const uint BLOCKDIM = 256;
extern "C" void spPostprocess2D(void *d_Dst, void *d_Src, uint DY, uint DX,
uint padding, int dir) {
assert(d_Src != d_Dst);
assert(DX % 2 == 0);
extern "C" void spPostprocess2D(void *d_Dst, void *d_Src, uint DY, uint DX, uint padding, int dir)
{
assert(d_Src != d_Dst);
assert(DX % 2 == 0);
#if (POWER_OF_TWO)
uint log2DX, log2DY;
uint factorizationRemX = factorRadix2(log2DX, DX);
uint factorizationRemY = factorRadix2(log2DY, DY);
assert(factorizationRemX == 1 && factorizationRemY == 1);
uint log2DX, log2DY;
uint factorizationRemX = factorRadix2(log2DX, DX);
uint factorizationRemY = factorRadix2(log2DY, DY);
assert(factorizationRemX == 1 && factorizationRemY == 1);
#endif
const uint threadCount = DY * (DX / 2);
const double phaseBase = dir * PI / (double)DX;
const uint threadCount = DY * (DX / 2);
const double phaseBase = dir * PI / (double)DX;
#if (USE_TEXTURE)
cudaTextureObject_t texComplex;
cudaResourceDesc texRes;
memset(&texRes, 0, sizeof(cudaResourceDesc));
cudaTextureObject_t texComplex;
cudaResourceDesc texRes;
memset(&texRes, 0, sizeof(cudaResourceDesc));
texRes.resType = cudaResourceTypeLinear;
texRes.res.linear.devPtr = d_Src;
texRes.res.linear.sizeInBytes = sizeof(fComplex) * DY * (DX + padding);
texRes.res.linear.desc = cudaCreateChannelDesc<fComplex>();
texRes.resType = cudaResourceTypeLinear;
texRes.res.linear.devPtr = d_Src;
texRes.res.linear.sizeInBytes = sizeof(fComplex) * DY * (DX + padding);
texRes.res.linear.desc = cudaCreateChannelDesc<fComplex>();
cudaTextureDesc texDescr;
memset(&texDescr, 0, sizeof(cudaTextureDesc));
cudaTextureDesc texDescr;
memset(&texDescr, 0, sizeof(cudaTextureDesc));
texDescr.normalizedCoords = false;
texDescr.filterMode = cudaFilterModeLinear;
texDescr.addressMode[0] = cudaAddressModeWrap;
texDescr.readMode = cudaReadModeElementType;
texDescr.normalizedCoords = false;
texDescr.filterMode = cudaFilterModeLinear;
texDescr.addressMode[0] = cudaAddressModeWrap;
texDescr.readMode = cudaReadModeElementType;
checkCudaErrors(
cudaCreateTextureObject(&texComplex, &texRes, &texDescr, NULL));
checkCudaErrors(cudaCreateTextureObject(&texComplex, &texRes, &texDescr, NULL));
#endif
spPostprocess2D_kernel<<<iDivUp(threadCount, BLOCKDIM), BLOCKDIM>>>(
(fComplex *)d_Dst, (fComplex *)d_Src, DY, DX, threadCount, padding,
(float)phaseBase
spPostprocess2D_kernel<<<iDivUp(threadCount, BLOCKDIM), BLOCKDIM>>>((fComplex *)d_Dst,
(fComplex *)d_Src,
DY,
DX,
threadCount,
padding,
(float)phaseBase
#if (USE_TEXTURE)
,
texComplex
,
texComplex
#endif
);
getLastCudaError("spPostprocess2D_kernel<<<>>> execution failed\n");
);
getLastCudaError("spPostprocess2D_kernel<<<>>> execution failed\n");
#if (USE_TEXTURE)
checkCudaErrors(cudaDestroyTextureObject(texComplex));
checkCudaErrors(cudaDestroyTextureObject(texComplex));
#endif
}
extern "C" void spPreprocess2D(void *d_Dst, void *d_Src, uint DY, uint DX,
uint padding, int dir) {
assert(d_Src != d_Dst);
assert(DX % 2 == 0);
extern "C" void spPreprocess2D(void *d_Dst, void *d_Src, uint DY, uint DX, uint padding, int dir)
{
assert(d_Src != d_Dst);
assert(DX % 2 == 0);
#if (POWER_OF_TWO)
uint log2DX, log2DY;
uint factorizationRemX = factorRadix2(log2DX, DX);
uint factorizationRemY = factorRadix2(log2DY, DY);
assert(factorizationRemX == 1 && factorizationRemY == 1);
uint log2DX, log2DY;
uint factorizationRemX = factorRadix2(log2DX, DX);
uint factorizationRemY = factorRadix2(log2DY, DY);
assert(factorizationRemX == 1 && factorizationRemY == 1);
#endif
const uint threadCount = DY * (DX / 2);
const double phaseBase = -dir * PI / (double)DX;
const uint threadCount = DY * (DX / 2);
const double phaseBase = -dir * PI / (double)DX;
#if (USE_TEXTURE)
cudaTextureObject_t texComplex;
cudaResourceDesc texRes;
memset(&texRes, 0, sizeof(cudaResourceDesc));
cudaTextureObject_t texComplex;
cudaResourceDesc texRes;
memset(&texRes, 0, sizeof(cudaResourceDesc));
texRes.resType = cudaResourceTypeLinear;
texRes.res.linear.devPtr = d_Src;
texRes.res.linear.sizeInBytes = sizeof(fComplex) * DY * (DX + padding);
texRes.res.linear.desc = cudaCreateChannelDesc<fComplex>();
texRes.resType = cudaResourceTypeLinear;
texRes.res.linear.devPtr = d_Src;
texRes.res.linear.sizeInBytes = sizeof(fComplex) * DY * (DX + padding);
texRes.res.linear.desc = cudaCreateChannelDesc<fComplex>();
cudaTextureDesc texDescr;
memset(&texDescr, 0, sizeof(cudaTextureDesc));
cudaTextureDesc texDescr;
memset(&texDescr, 0, sizeof(cudaTextureDesc));
texDescr.normalizedCoords = false;
texDescr.filterMode = cudaFilterModeLinear;
texDescr.addressMode[0] = cudaAddressModeWrap;
texDescr.readMode = cudaReadModeElementType;
texDescr.normalizedCoords = false;
texDescr.filterMode = cudaFilterModeLinear;
texDescr.addressMode[0] = cudaAddressModeWrap;
texDescr.readMode = cudaReadModeElementType;
checkCudaErrors(
cudaCreateTextureObject(&texComplex, &texRes, &texDescr, NULL));
checkCudaErrors(cudaCreateTextureObject(&texComplex, &texRes, &texDescr, NULL));
#endif
spPreprocess2D_kernel<<<iDivUp(threadCount, BLOCKDIM), BLOCKDIM>>>(
(fComplex *)d_Dst, (fComplex *)d_Src, DY, DX, threadCount, padding,
(float)phaseBase
spPreprocess2D_kernel<<<iDivUp(threadCount, BLOCKDIM), BLOCKDIM>>>((fComplex *)d_Dst,
(fComplex *)d_Src,
DY,
DX,
threadCount,
padding,
(float)phaseBase
#if (USE_TEXTURE)
,
texComplex
,
texComplex
#endif
);
getLastCudaError("spPreprocess2D_kernel<<<>>> execution failed\n");
);
getLastCudaError("spPreprocess2D_kernel<<<>>> execution failed\n");
#if (USE_TEXTURE)
checkCudaErrors(cudaDestroyTextureObject(texComplex));
checkCudaErrors(cudaDestroyTextureObject(texComplex));
#endif
}
////////////////////////////////////////////////////////////////////////////////
// Combined spPostprocess2D + modulateAndNormalize + spPreprocess2D
////////////////////////////////////////////////////////////////////////////////
extern "C" void spProcess2D(void *d_Dst, void *d_SrcA, void *d_SrcB, uint DY,
uint DX, int dir) {
assert(DY % 2 == 0);
extern "C" void spProcess2D(void *d_Dst, void *d_SrcA, void *d_SrcB, uint DY, uint DX, int dir)
{
assert(DY % 2 == 0);
#if (POWER_OF_TWO)
uint log2DX, log2DY;
uint factorizationRemX = factorRadix2(log2DX, DX);
uint factorizationRemY = factorRadix2(log2DY, DY);
assert(factorizationRemX == 1 && factorizationRemY == 1);
uint log2DX, log2DY;
uint factorizationRemX = factorRadix2(log2DX, DX);
uint factorizationRemY = factorRadix2(log2DY, DY);
assert(factorizationRemX == 1 && factorizationRemY == 1);
#endif
const uint threadCount = (DY / 2) * DX;
const double phaseBase = dir * PI / (double)DX;
const uint threadCount = (DY / 2) * DX;
const double phaseBase = dir * PI / (double)DX;
#if (USE_TEXTURE)
cudaTextureObject_t texComplexA, texComplexB;
cudaResourceDesc texRes;
memset(&texRes, 0, sizeof(cudaResourceDesc));
cudaTextureObject_t texComplexA, texComplexB;
cudaResourceDesc texRes;
memset(&texRes, 0, sizeof(cudaResourceDesc));
texRes.resType = cudaResourceTypeLinear;
texRes.res.linear.devPtr = d_SrcA;
texRes.res.linear.sizeInBytes = sizeof(fComplex) * DY * DX;
texRes.res.linear.desc = cudaCreateChannelDesc<fComplex>();
texRes.resType = cudaResourceTypeLinear;
texRes.res.linear.devPtr = d_SrcA;
texRes.res.linear.sizeInBytes = sizeof(fComplex) * DY * DX;
texRes.res.linear.desc = cudaCreateChannelDesc<fComplex>();
cudaTextureDesc texDescr;
memset(&texDescr, 0, sizeof(cudaTextureDesc));
cudaTextureDesc texDescr;
memset(&texDescr, 0, sizeof(cudaTextureDesc));
texDescr.normalizedCoords = false;
texDescr.filterMode = cudaFilterModeLinear;
texDescr.addressMode[0] = cudaAddressModeWrap;
texDescr.readMode = cudaReadModeElementType;
texDescr.normalizedCoords = false;
texDescr.filterMode = cudaFilterModeLinear;
texDescr.addressMode[0] = cudaAddressModeWrap;
texDescr.readMode = cudaReadModeElementType;
checkCudaErrors(
cudaCreateTextureObject(&texComplexA, &texRes, &texDescr, NULL));
checkCudaErrors(cudaCreateTextureObject(&texComplexA, &texRes, &texDescr, NULL));
memset(&texRes, 0, sizeof(cudaResourceDesc));
memset(&texRes, 0, sizeof(cudaResourceDesc));
texRes.resType = cudaResourceTypeLinear;
texRes.res.linear.devPtr = d_SrcB;
texRes.res.linear.sizeInBytes = sizeof(fComplex) * DY * DX;
texRes.res.linear.desc = cudaCreateChannelDesc<fComplex>();
texRes.resType = cudaResourceTypeLinear;
texRes.res.linear.devPtr = d_SrcB;
texRes.res.linear.sizeInBytes = sizeof(fComplex) * DY * DX;
texRes.res.linear.desc = cudaCreateChannelDesc<fComplex>();
memset(&texDescr, 0, sizeof(cudaTextureDesc));
memset(&texDescr, 0, sizeof(cudaTextureDesc));
texDescr.normalizedCoords = false;
texDescr.filterMode = cudaFilterModeLinear;
texDescr.addressMode[0] = cudaAddressModeWrap;
texDescr.readMode = cudaReadModeElementType;
texDescr.normalizedCoords = false;
texDescr.filterMode = cudaFilterModeLinear;
texDescr.addressMode[0] = cudaAddressModeWrap;
texDescr.readMode = cudaReadModeElementType;
checkCudaErrors(
cudaCreateTextureObject(&texComplexB, &texRes, &texDescr, NULL));
checkCudaErrors(cudaCreateTextureObject(&texComplexB, &texRes, &texDescr, NULL));
#endif
spProcess2D_kernel<<<iDivUp(threadCount, BLOCKDIM), BLOCKDIM>>>(
(fComplex *)d_Dst, (fComplex *)d_SrcA, (fComplex *)d_SrcB, DY, DX,
threadCount, (float)phaseBase, 0.5f / (float)(DY * DX)
spProcess2D_kernel<<<iDivUp(threadCount, BLOCKDIM), BLOCKDIM>>>((fComplex *)d_Dst,
(fComplex *)d_SrcA,
(fComplex *)d_SrcB,
DY,
DX,
threadCount,
(float)phaseBase,
0.5f / (float)(DY * DX)
#if (USE_TEXTURE)
,
texComplexA, texComplexB
,
texComplexA,
texComplexB
#endif
);
getLastCudaError("spProcess2D_kernel<<<>>> execution failed\n");
);
getLastCudaError("spProcess2D_kernel<<<>>> execution failed\n");
#if (USE_TEXTURE)
checkCudaErrors(cudaDestroyTextureObject(texComplexA));
checkCudaErrors(cudaDestroyTextureObject(texComplexB));
checkCudaErrors(cudaDestroyTextureObject(texComplexA));
checkCudaErrors(cudaDestroyTextureObject(texComplexB));
#endif
}

543
Samples/5_Domain_Specific/convolutionFFT2D/convolutionFFT2D.cuh
View File

@@ -25,7 +25,7 @@
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#define USE_TEXTURE 1
#define USE_TEXTURE 1
#define POWER_OF_TWO 1
#if (USE_TEXTURE)
@@ -36,116 +36,131 @@
#define SET_FLOAT_BASE
#endif
#include "convolutionFFT2D_common.h"
////////////////////////////////////////////////////////////////////////////////
/// Position convolution kernel center at (0, 0) in the image
////////////////////////////////////////////////////////////////////////////////
__global__ void padKernel_kernel(float *d_Dst, float *d_Src, int fftH, int fftW,
int kernelH, int kernelW, int kernelY,
int kernelX
__global__ void padKernel_kernel(float *d_Dst,
float *d_Src,
int fftH,
int fftW,
int kernelH,
int kernelW,
int kernelY,
int kernelX
#if (USE_TEXTURE)
,
cudaTextureObject_t texFloat
#endif
) {
const int y = blockDim.y * blockIdx.y + threadIdx.y;
const int x = blockDim.x * blockIdx.x + threadIdx.x;
)
{
const int y = blockDim.y * blockIdx.y + threadIdx.y;
const int x = blockDim.x * blockIdx.x + threadIdx.x;
if (y < kernelH && x < kernelW) {
int ky = y - kernelY;
if (y < kernelH && x < kernelW) {
int ky = y - kernelY;
if (ky < 0) {
ky += fftH;
if (ky < 0) {
ky += fftH;
}
int kx = x - kernelX;
if (kx < 0) {
kx += fftW;
}
d_Dst[ky * fftW + kx] = LOAD_FLOAT(y * kernelW + x);
}
int kx = x - kernelX;
if (kx < 0) {
kx += fftW;
}
d_Dst[ky * fftW + kx] = LOAD_FLOAT(y * kernelW + x);
}
}
////////////////////////////////////////////////////////////////////////////////
// Prepare data for "pad to border" addressing mode
////////////////////////////////////////////////////////////////////////////////
__global__ void padDataClampToBorder_kernel(float *d_Dst, float *d_Src,
int fftH, int fftW, int dataH,
int dataW, int kernelH, int kernelW,
int kernelY, int kernelX
__global__ void padDataClampToBorder_kernel(float *d_Dst,
float *d_Src,
int fftH,
int fftW,
int dataH,
int dataW,
int kernelH,
int kernelW,
int kernelY,
int kernelX
#if (USE_TEXTURE)
,
cudaTextureObject_t texFloat
#endif
) {
const int y = blockDim.y * blockIdx.y + threadIdx.y;
const int x = blockDim.x * blockIdx.x + threadIdx.x;
const int borderH = dataH + kernelY;
const int borderW = dataW + kernelX;
)
{
const int y = blockDim.y * blockIdx.y + threadIdx.y;
const int x = blockDim.x * blockIdx.x + threadIdx.x;
const int borderH = dataH + kernelY;
const int borderW = dataW + kernelX;
if (y < fftH && x < fftW) {
int dy, dx;
if (y < fftH && x < fftW) {
int dy, dx;
if (y < dataH) {
dy = y;
if (y < dataH) {
dy = y;
}
if (x < dataW) {
dx = x;
}
if (y >= dataH && y < borderH) {
dy = dataH - 1;
}
if (x >= dataW && x < borderW) {
dx = dataW - 1;
}
if (y >= borderH) {
dy = 0;
}
if (x >= borderW) {
dx = 0;
}
d_Dst[y * fftW + x] = LOAD_FLOAT(dy * dataW + dx);
}
if (x < dataW) {
dx = x;
}
if (y >= dataH && y < borderH) {
dy = dataH - 1;
}
if (x >= dataW && x < borderW) {
dx = dataW - 1;
}
if (y >= borderH) {
dy = 0;
}
if (x >= borderW) {
dx = 0;
}
d_Dst[y * fftW + x] = LOAD_FLOAT(dy * dataW + dx);
}
}
////////////////////////////////////////////////////////////////////////////////
// Modulate Fourier image of padded data by Fourier image of padded kernel
// and normalize by FFT size
////////////////////////////////////////////////////////////////////////////////
inline __device__ void mulAndScale(fComplex &a, const fComplex &b,
const float &c) {
fComplex t = {c * (a.x * b.x - a.y * b.y), c * (a.y * b.x + a.x * b.y)};
a = t;
inline __device__ void mulAndScale(fComplex &a, const fComplex &b, const float &c)
{
fComplex t = {c * (a.x * b.x - a.y * b.y), c * (a.y * b.x + a.x * b.y)};
a = t;
}
__global__ void modulateAndNormalize_kernel(fComplex *d_Dst, fComplex *d_Src,
int dataSize, float c) {
const int i = blockDim.x * blockIdx.x + threadIdx.x;
__global__ void modulateAndNormalize_kernel(fComplex *d_Dst, fComplex *d_Src, int dataSize, float c)
{
const int i = blockDim.x * blockIdx.x + threadIdx.x;
if (i >= dataSize) {
return;
}
if (i >= dataSize) {
return;
}
fComplex a = d_Src[i];
fComplex b = d_Dst[i];
fComplex a = d_Src[i];
fComplex b = d_Dst[i];
mulAndScale(a, b, c);
mulAndScale(a, b, c);
d_Dst[i] = a;
d_Dst[i] = a;
}
////////////////////////////////////////////////////////////////////////////////
// 2D R2C / C2R post/preprocessing kernels
////////////////////////////////////////////////////////////////////////////////
#if (USE_TEXTURE)
#define LOAD_FCOMPLEX(i) tex1Dfetch<fComplex>(texComplex, i)
#define LOAD_FCOMPLEX(i) tex1Dfetch<fComplex>(texComplex, i)
#define LOAD_FCOMPLEX_A(i) tex1Dfetch<fComplex>(texComplexA, i)
#define LOAD_FCOMPLEX_B(i) tex1Dfetch<fComplex>(texComplexB, i)
@@ -153,7 +168,7 @@ __global__ void modulateAndNormalize_kernel(fComplex *d_Dst, fComplex *d_Src,
#define SET_FCOMPLEX_BASE_A
#define SET_FCOMPLEX_BASE_B
#else
#define LOAD_FCOMPLEX(i) d_Src[i]
#define LOAD_FCOMPLEX(i) d_Src[i]
#define LOAD_FCOMPLEX_A(i) d_SrcA[i]
#define LOAD_FCOMPLEX_B(i) d_SrcB[i]
@@ -162,260 +177,278 @@ __global__ void modulateAndNormalize_kernel(fComplex *d_Dst, fComplex *d_Src,
#define SET_FCOMPLEX_BASE_B
#endif
inline __device__ void spPostprocessC2C(fComplex &D1, fComplex &D2,
const fComplex &twiddle) {
float A1 = 0.5f * (D1.x + D2.x);
float B1 = 0.5f * (D1.y - D2.y);
float A2 = 0.5f * (D1.y + D2.y);
float B2 = 0.5f * (D1.x - D2.x);
inline __device__ void spPostprocessC2C(fComplex &D1, fComplex &D2, const fComplex &twiddle)
{
float A1 = 0.5f * (D1.x + D2.x);
float B1 = 0.5f * (D1.y - D2.y);
float A2 = 0.5f * (D1.y + D2.y);
float B2 = 0.5f * (D1.x - D2.x);
D1.x = A1 + (A2 * twiddle.x + B2 * twiddle.y);
D1.y = (A2 * twiddle.y - B2 * twiddle.x) + B1;
D2.x = A1 - (A2 * twiddle.x + B2 * twiddle.y);
D2.y = (A2 * twiddle.y - B2 * twiddle.x) - B1;
D1.x = A1 + (A2 * twiddle.x + B2 * twiddle.y);
D1.y = (A2 * twiddle.y - B2 * twiddle.x) + B1;
D2.x = A1 - (A2 * twiddle.x + B2 * twiddle.y);
D2.y = (A2 * twiddle.y - B2 * twiddle.x) - B1;
}
// Premultiply by 2 to account for 1.0 / (DZ * DY * DX) normalization
inline __device__ void spPreprocessC2C(fComplex &D1, fComplex &D2,
const fComplex &twiddle) {
float A1 = /* 0.5f * */ (D1.x + D2.x);
float B1 = /* 0.5f * */ (D1.y - D2.y);
float A2 = /* 0.5f * */ (D1.y + D2.y);
float B2 = /* 0.5f * */ (D1.x - D2.x);
inline __device__ void spPreprocessC2C(fComplex &D1, fComplex &D2, const fComplex &twiddle)
{
float A1 = /* 0.5f * */ (D1.x + D2.x);
float B1 = /* 0.5f * */ (D1.y - D2.y);
float A2 = /* 0.5f * */ (D1.y + D2.y);
float B2 = /* 0.5f * */ (D1.x - D2.x);
D1.x = A1 - (A2 * twiddle.x - B2 * twiddle.y);
D1.y = (B2 * twiddle.x + A2 * twiddle.y) + B1;
D2.x = A1 + (A2 * twiddle.x - B2 * twiddle.y);
D2.y = (B2 * twiddle.x + A2 * twiddle.y) - B1;
D1.x = A1 - (A2 * twiddle.x - B2 * twiddle.y);
D1.y = (B2 * twiddle.x + A2 * twiddle.y) + B1;
D2.x = A1 + (A2 * twiddle.x - B2 * twiddle.y);
D2.y = (B2 * twiddle.x + A2 * twiddle.y) - B1;
}
inline __device__ void getTwiddle(fComplex &twiddle, float phase) {
__sincosf(phase, &twiddle.y, &twiddle.x);
inline __device__ void getTwiddle(fComplex &twiddle, float phase) { __sincosf(phase, &twiddle.y, &twiddle.x); }
inline __device__ uint mod(uint a, uint DA)
{
//(DA - a) % DA, assuming a <= DA
return a ? (DA - a) : a;
}
inline __device__ uint mod(uint a, uint DA) {
//(DA - a) % DA, assuming a <= DA
return a ? (DA - a) : a;
static inline uint factorRadix2(uint &log2N, uint n)
{
if (!n) {
log2N = 0;
return 0;
}
else {
for (log2N = 0; n % 2 == 0; n /= 2, log2N++)
;
return n;
}
}
static inline uint factorRadix2(uint &log2N, uint n) {
if (!n) {
log2N = 0;
return 0;
} else {
for (log2N = 0; n % 2 == 0; n /= 2, log2N++)
;
return n;
}
}
inline __device__ void udivmod(uint &dividend, uint divisor, uint &rem) {
inline __device__ void udivmod(uint &dividend, uint divisor, uint &rem)
{
#if (!POWER_OF_TWO)
rem = dividend % divisor;
dividend /= divisor;
rem = dividend % divisor;
dividend /= divisor;
#else
rem = dividend & (divisor - 1);
dividend >>= (__ffs(divisor) - 1);
rem = dividend & (divisor - 1);
dividend >>= (__ffs(divisor) - 1);
#endif
}
__global__ void spPostprocess2D_kernel(fComplex *d_Dst, fComplex *d_Src,
uint DY, uint DX, uint threadCount,
uint padding, float phaseBase
__global__ void spPostprocess2D_kernel(fComplex *d_Dst,
fComplex *d_Src,
uint DY,
uint DX,
uint threadCount,
uint padding,
float phaseBase
#if (USE_TEXTURE)
,
cudaTextureObject_t texComplex
#endif
) {
const uint threadId = blockIdx.x * blockDim.x + threadIdx.x;
)
{
const uint threadId = blockIdx.x * blockDim.x + threadIdx.x;
if (threadId >= threadCount) {
return;
}
if (threadId >= threadCount) {
return;
}
uint x, y, i = threadId;
udivmod(i, DX / 2, x);
udivmod(i, DY, y);
uint x, y, i = threadId;
udivmod(i, DX / 2, x);
udivmod(i, DY, y);
// Avoid overwrites in columns DX / 2 by different threads
if ((x == 0) && (y > DY / 2)) {
return;
}
// Avoid overwrites in columns DX / 2 by different threads
if ((x == 0) && (y > DY / 2)) {
return;
}
const uint srcOffset = i * DY * DX;
const uint dstOffset = i * DY * (DX + padding);
const uint srcOffset = i * DY * DX;
const uint dstOffset = i * DY * (DX + padding);
// Process x = [0 .. DX / 2 - 1] U [DX / 2 + 1 .. DX]
{
const uint loadPos1 = srcOffset + y * DX + x;
const uint loadPos2 = srcOffset + mod(y, DY) * DX + mod(x, DX);
const uint storePos1 = dstOffset + y * (DX + padding) + x;
const uint storePos2 = dstOffset + mod(y, DY) * (DX + padding) + (DX - x);
// Process x = [0 .. DX / 2 - 1] U [DX / 2 + 1 .. DX]
{
const uint loadPos1 = srcOffset + y * DX + x;
const uint loadPos2 = srcOffset + mod(y, DY) * DX + mod(x, DX);
const uint storePos1 = dstOffset + y * (DX + padding) + x;
const uint storePos2 = dstOffset + mod(y, DY) * (DX + padding) + (DX - x);
fComplex D1 = LOAD_FCOMPLEX(loadPos1);
fComplex D2 = LOAD_FCOMPLEX(loadPos2);
fComplex D1 = LOAD_FCOMPLEX(loadPos1);
fComplex D2 = LOAD_FCOMPLEX(loadPos2);
fComplex twiddle;
getTwiddle(twiddle, phaseBase * (float)x);
spPostprocessC2C(D1, D2, twiddle);
fComplex twiddle;
getTwiddle(twiddle, phaseBase * (float)x);
spPostprocessC2C(D1, D2, twiddle);
d_Dst[storePos1] = D1;
d_Dst[storePos2] = D2;
}
d_Dst[storePos1] = D1;
d_Dst[storePos2] = D2;
}
// Process x = DX / 2
if (x == 0) {
const uint loadPos1 = srcOffset + y * DX + DX / 2;
const uint loadPos2 = srcOffset + mod(y, DY) * DX + DX / 2;
const uint storePos1 = dstOffset + y * (DX + padding) + DX / 2;
const uint storePos2 = dstOffset + mod(y, DY) * (DX + padding) + DX / 2;
// Process x = DX / 2
if (x == 0) {
const uint loadPos1 = srcOffset + y * DX + DX / 2;
const uint loadPos2 = srcOffset + mod(y, DY) * DX + DX / 2;
const uint storePos1 = dstOffset + y * (DX + padding) + DX / 2;
const uint storePos2 = dstOffset + mod(y, DY) * (DX + padding) + DX / 2;
fComplex D1 = LOAD_FCOMPLEX(loadPos1);
fComplex D2 = LOAD_FCOMPLEX(loadPos2);
fComplex D1 = LOAD_FCOMPLEX(loadPos1);
fComplex D2 = LOAD_FCOMPLEX(loadPos2);
// twiddle = getTwiddle(phaseBase * (DX / 2)) = exp(dir * j * PI / 2)
fComplex twiddle = {0, (phaseBase > 0) ? 1.0f : -1.0f};
spPostprocessC2C(D1, D2, twiddle);
// twiddle = getTwiddle(phaseBase * (DX / 2)) = exp(dir * j * PI / 2)
fComplex twiddle = {0, (phaseBase > 0) ? 1.0f : -1.0f};
spPostprocessC2C(D1, D2, twiddle);
d_Dst[storePos1] = D1;
d_Dst[storePos2] = D2;
}
d_Dst[storePos1] = D1;
d_Dst[storePos2] = D2;
}
}
__global__ void spPreprocess2D_kernel(fComplex *d_Dst, fComplex *d_Src, uint DY,
uint DX, uint threadCount, uint padding,
float phaseBase
__global__ void spPreprocess2D_kernel(fComplex *d_Dst,
fComplex *d_Src,
uint DY,
uint DX,
uint threadCount,
uint padding,
float phaseBase
#if (USE_TEXTURE)
,
cudaTextureObject_t texComplex
#endif
) {
const uint threadId = blockIdx.x * blockDim.x + threadIdx.x;
)
{
const uint threadId = blockIdx.x * blockDim.x + threadIdx.x;
if (threadId >= threadCount) {
return;
}
if (threadId >= threadCount) {
return;
}
uint x, y, i = threadId;
udivmod(i, DX / 2, x);
udivmod(i, DY, y);
uint x, y, i = threadId;
udivmod(i, DX / 2, x);
udivmod(i, DY, y);
// Avoid overwrites in columns 0 and DX / 2 by different threads (lower and
// upper halves)
if ((x == 0) && (y > DY / 2)) {
return;
}
// Avoid overwrites in columns 0 and DX / 2 by different threads (lower and
// upper halves)
if ((x == 0) && (y > DY / 2)) {
return;
}
const uint srcOffset = i * DY * (DX + padding);
const uint dstOffset = i * DY * DX;
const uint srcOffset = i * DY * (DX + padding);
const uint dstOffset = i * DY * DX;
// Process x = [0 .. DX / 2 - 1] U [DX / 2 + 1 .. DX]
{
const uint loadPos1 = srcOffset + y * (DX + padding) + x;
const uint loadPos2 = srcOffset + mod(y, DY) * (DX + padding) + (DX - x);
const uint storePos1 = dstOffset + y * DX + x;
const uint storePos2 = dstOffset + mod(y, DY) * DX + mod(x, DX);
// Process x = [0 .. DX / 2 - 1] U [DX / 2 + 1 .. DX]
{
const uint loadPos1 = srcOffset + y * (DX + padding) + x;
const uint loadPos2 = srcOffset + mod(y, DY) * (DX + padding) + (DX - x);
const uint storePos1 = dstOffset + y * DX + x;
const uint storePos2 = dstOffset + mod(y, DY) * DX + mod(x, DX);
fComplex D1 = LOAD_FCOMPLEX(loadPos1);
fComplex D2 = LOAD_FCOMPLEX(loadPos2);
fComplex D1 = LOAD_FCOMPLEX(loadPos1);
fComplex D2 = LOAD_FCOMPLEX(loadPos2);
fComplex twiddle;
getTwiddle(twiddle, phaseBase * (float)x);
spPreprocessC2C(D1, D2, twiddle);
fComplex twiddle;
getTwiddle(twiddle, phaseBase * (float)x);
spPreprocessC2C(D1, D2, twiddle);
d_Dst[storePos1] = D1;
d_Dst[storePos2] = D2;
}
d_Dst[storePos1] = D1;
d_Dst[storePos2] = D2;
}
// Process x = DX / 2
if (x == 0) {
const uint loadPos1 = srcOffset + y * (DX + padding) + DX / 2;
const uint loadPos2 = srcOffset + mod(y, DY) * (DX + padding) + DX / 2;
const uint storePos1 = dstOffset + y * DX + DX / 2;
const uint storePos2 = dstOffset + mod(y, DY) * DX + DX / 2;
// Process x = DX / 2
if (x == 0) {
const uint loadPos1 = srcOffset + y * (DX + padding) + DX / 2;
const uint loadPos2 = srcOffset + mod(y, DY) * (DX + padding) + DX / 2;
const uint storePos1 = dstOffset + y * DX + DX / 2;
const uint storePos2 = dstOffset + mod(y, DY) * DX + DX / 2;
fComplex D1 = LOAD_FCOMPLEX(loadPos1);
fComplex D2 = LOAD_FCOMPLEX(loadPos2);
fComplex D1 = LOAD_FCOMPLEX(loadPos1);
fComplex D2 = LOAD_FCOMPLEX(loadPos2);
// twiddle = getTwiddle(phaseBase * (DX / 2)) = exp(-dir * j * PI / 2)
fComplex twiddle = {0, (phaseBase > 0) ? 1.0f : -1.0f};
spPreprocessC2C(D1, D2, twiddle);
// twiddle = getTwiddle(phaseBase * (DX / 2)) = exp(-dir * j * PI / 2)
fComplex twiddle = {0, (phaseBase > 0) ? 1.0f : -1.0f};
spPreprocessC2C(D1, D2, twiddle);
d_Dst[storePos1] = D1;
d_Dst[storePos2] = D2;
}
d_Dst[storePos1] = D1;
d_Dst[storePos2] = D2;
}
}
////////////////////////////////////////////////////////////////////////////////
// Combined spPostprocess2D + modulateAndNormalize + spPreprocess2D
////////////////////////////////////////////////////////////////////////////////
__global__ void spProcess2D_kernel(fComplex *d_Dst, fComplex *d_SrcA,
fComplex *d_SrcB, uint DY, uint DX,
uint threadCount, float phaseBase, float c
__global__ void spProcess2D_kernel(fComplex *d_Dst,
fComplex *d_SrcA,
fComplex *d_SrcB,
uint DY,
uint DX,
uint threadCount,
float phaseBase,
float c
#if (USE_TEXTURE)
,
cudaTextureObject_t texComplexA,
cudaTextureObject_t texComplexB
#endif
) {
const uint threadId = blockIdx.x * blockDim.x + threadIdx.x;
)
{
const uint threadId = blockIdx.x * blockDim.x + threadIdx.x;
if (threadId >= threadCount) {
return;
}
if (threadId >= threadCount) {
return;
}
uint x, y, i = threadId;
udivmod(i, DX, x);
udivmod(i, DY / 2, y);
uint x, y, i = threadId;
udivmod(i, DX, x);
udivmod(i, DY / 2, y);
const uint offset = i * DY * DX;
const uint offset = i * DY * DX;
// Avoid overwrites in rows 0 and DY / 2 by different threads (left and right
// halves) Otherwise correctness for in-place transformations is affected
if ((y == 0) && (x > DX / 2)) {
return;
}
// Avoid overwrites in rows 0 and DY / 2 by different threads (left and right
// halves) Otherwise correctness for in-place transformations is affected
if ((y == 0) && (x > DX / 2)) {
return;
}
fComplex twiddle;
fComplex twiddle;
// Process y = [0 .. DY / 2 - 1] U [DY - (DY / 2) + 1 .. DY - 1]
{
const uint pos1 = offset + y * DX + x;
const uint pos2 = offset + mod(y, DY) * DX + mod(x, DX);
// Process y = [0 .. DY / 2 - 1] U [DY - (DY / 2) + 1 .. DY - 1]
{
const uint pos1 = offset + y * DX + x;
const uint pos2 = offset + mod(y, DY) * DX + mod(x, DX);
fComplex D1 = LOAD_FCOMPLEX_A(pos1);
fComplex D2 = LOAD_FCOMPLEX_A(pos2);
fComplex K1 = LOAD_FCOMPLEX_B(pos1);
fComplex K2 = LOAD_FCOMPLEX_B(pos2);
getTwiddle(twiddle, phaseBase * (float)x);
fComplex D1 = LOAD_FCOMPLEX_A(pos1);
fComplex D2 = LOAD_FCOMPLEX_A(pos2);
fComplex K1 = LOAD_FCOMPLEX_B(pos1);
fComplex K2 = LOAD_FCOMPLEX_B(pos2);
getTwiddle(twiddle, phaseBase * (float)x);
spPostprocessC2C(D1, D2, twiddle);
spPostprocessC2C(K1, K2, twiddle);
mulAndScale(D1, K1, c);
mulAndScale(D2, K2, c);
spPreprocessC2C(D1, D2, twiddle);
spPostprocessC2C(D1, D2, twiddle);
spPostprocessC2C(K1, K2, twiddle);
mulAndScale(D1, K1, c);
mulAndScale(D2, K2, c);
spPreprocessC2C(D1, D2, twiddle);
d_Dst[pos1] = D1;
d_Dst[pos2] = D2;
}
d_Dst[pos1] = D1;
d_Dst[pos2] = D2;
}
if (y == 0) {
const uint pos1 = offset + (DY / 2) * DX + x;
const uint pos2 = offset + (DY / 2) * DX + mod(x, DX);
if (y == 0) {
const uint pos1 = offset + (DY / 2) * DX + x;
const uint pos2 = offset + (DY / 2) * DX + mod(x, DX);
fComplex D1 = LOAD_FCOMPLEX_A(pos1);
fComplex D2 = LOAD_FCOMPLEX_A(pos2);
fComplex K1 = LOAD_FCOMPLEX_B(pos1);
fComplex K2 = LOAD_FCOMPLEX_B(pos2);
fComplex D1 = LOAD_FCOMPLEX_A(pos1);
fComplex D2 = LOAD_FCOMPLEX_A(pos2);
fComplex K1 = LOAD_FCOMPLEX_B(pos1);
fComplex K2 = LOAD_FCOMPLEX_B(pos2);
spPostprocessC2C(D1, D2, twiddle);
spPostprocessC2C(K1, K2, twiddle);
mulAndScale(D1, K1, c);
mulAndScale(D2, K2, c);
spPreprocessC2C(D1, D2, twiddle);
spPostprocessC2C(D1, D2, twiddle);
spPostprocessC2C(K1, K2, twiddle);
mulAndScale(D1, K1, c);
mulAndScale(D2, K2, c);
spPreprocessC2C(D1, D2, twiddle);
d_Dst[pos1] = D1;
d_Dst[pos2] = D2;
}
d_Dst[pos1] = D1;
d_Dst[pos2] = D2;
}
}

59
Samples/5_Domain_Specific/convolutionFFT2D/convolutionFFT2D_common.h
View File

@@ -33,9 +33,10 @@ typedef unsigned int uint;
#ifdef __CUDACC__
typedef float2 fComplex;
#else
typedef struct {
float x;
float y;
typedef struct
{
float x;
float y;
} fComplex;
#endif
@@ -48,30 +49,42 @@ inline int iDivUp(int a, int b) { return (a % b != 0) ? (a / b + 1) : (a / b); }
// Align a to nearest higher multiple of b
inline int iAlignUp(int a, int b) { return (a % b != 0) ? (a - a % b + b) : a; }
extern "C" void convolutionClampToBorderCPU(float *h_Result, float *h_Data,
float *h_Kernel, int dataH,
int dataW, int kernelH, int kernelW,
int kernelY, int kernelX);
extern "C" void convolutionClampToBorderCPU(float *h_Result,
float *h_Data,
float *h_Kernel,
int dataH,
int dataW,
int kernelH,
int kernelW,
int kernelY,
int kernelX);
extern "C" void padKernel(float *d_PaddedKernel, float *d_Kernel, int fftH,
int fftW, int kernelH, int kernelW, int kernelY,
int kernelX);
extern "C" void padKernel(float *d_PaddedKernel,
float *d_Kernel,
int fftH,
int fftW,
int kernelH,
int kernelW,
int kernelY,
int kernelX);
extern "C" void padDataClampToBorder(float *d_PaddedData, float *d_Data,
int fftH, int fftW, int dataH, int dataW,
int kernelH, int kernelW, int kernelY,
int kernelX);
extern "C" void padDataClampToBorder(float *d_PaddedData,
float *d_Data,
int fftH,
int fftW,
int dataH,
int dataW,
int kernelH,
int kernelW,
int kernelY,
int kernelX);
extern "C" void modulateAndNormalize(fComplex *d_Dst, fComplex *d_Src, int fftH,
int fftW, int padding);
extern "C" void modulateAndNormalize(fComplex *d_Dst, fComplex *d_Src, int fftH, int fftW, int padding);
extern "C" void spPostprocess2D(void *d_Dst, void *d_Src, uint DY, uint DX,
uint padding, int dir);
extern "C" void spPostprocess2D(void *d_Dst, void *d_Src, uint DY, uint DX, uint padding, int dir);
extern "C" void spPreprocess2D(void *d_Dst, void *d_Src, uint DY, uint DX,
uint padding, int dir);
extern "C" void spPreprocess2D(void *d_Dst, void *d_Src, uint DY, uint DX, uint padding, int dir);
extern "C" void spProcess2D(void *d_Data, void *d_Data0, void *d_Kernel0,
uint DY, uint DX, int dir);
extern "C" void spProcess2D(void *d_Data, void *d_Data0, void *d_Kernel0, uint DY, uint DX, int dir);
#endif // CONVOLUTIONFFT2D_COMMON_H
#endif // CONVOLUTIONFFT2D_COMMON_H

50
Samples/5_Domain_Specific/convolutionFFT2D/convolutionFFT2D_gold.cpp
View File

@@ -26,36 +26,46 @@
*/
#include <assert.h>
#include "convolutionFFT2D_common.h"
////////////////////////////////////////////////////////////////////////////////
// Reference straightforward CPU convolution
////////////////////////////////////////////////////////////////////////////////
extern "C" void convolutionClampToBorderCPU(float *h_Result, float *h_Data,
float *h_Kernel, int dataH,
int dataW, int kernelH, int kernelW,
int kernelY, int kernelX) {
for (int y = 0; y < dataH; y++)
for (int x = 0; x < dataW; x++) {
double sum = 0;
extern "C" void convolutionClampToBorderCPU(float *h_Result,
float *h_Data,
float *h_Kernel,
int dataH,
int dataW,
int kernelH,
int kernelW,
int kernelY,
int kernelX)
{
for (int y = 0; y < dataH; y++)
for (int x = 0; x < dataW; x++) {
double sum = 0;
for (int ky = -(kernelH - kernelY - 1); ky <= kernelY; ky++)
for (int kx = -(kernelW - kernelX - 1); kx <= kernelX; kx++) {
int dy = y + ky;
int dx = x + kx;
for (int ky = -(kernelH - kernelY - 1); ky <= kernelY; ky++)
for (int kx = -(kernelW - kernelX - 1); kx <= kernelX; kx++) {
int dy = y + ky;
int dx = x + kx;
if (dy < 0) dy = 0;
if (dy < 0)
dy = 0;
if (dx < 0) dx = 0;
if (dx < 0)
dx = 0;
if (dy >= dataH) dy = dataH - 1;
if (dy >= dataH)
dy = dataH - 1;
if (dx >= dataW) dx = dataW - 1;
if (dx >= dataW)
dx = dataW - 1;
sum += h_Data[dy * dataW + dx] *
h_Kernel[(kernelY - ky) * kernelW + (kernelX - kx)];
sum += h_Data[dy * dataW + dx] * h_Kernel[(kernelY - ky) * kernelW + (kernelX - kx)];
}
h_Result[y * dataW + x] = (float)sum;
}
h_Result[y * dataW + x] = (float)sum;
}
}

898
Samples/5_Domain_Specific/convolutionFFT2D/main.cpp
View File

@@ -42,545 +42,489 @@
#include <cufft.h>
// Helper functions for CUDA
#include <helper_functions.h>
#include <helper_cuda.h>
#include <helper_functions.h>
#include "convolutionFFT2D_common.h"
////////////////////////////////////////////////////////////////////////////////
// Helper functions
////////////////////////////////////////////////////////////////////////////////
int snapTransformSize(int dataSize) {
int hiBit;
unsigned int lowPOT, hiPOT;
int snapTransformSize(int dataSize)
{
int hiBit;
unsigned int lowPOT, hiPOT;
dataSize = iAlignUp(dataSize, 16);
dataSize = iAlignUp(dataSize, 16);
for (hiBit = 31; hiBit >= 0; hiBit--)
if (dataSize & (1U << hiBit)) {
break;
for (hiBit = 31; hiBit >= 0; hiBit--)
if (dataSize & (1U << hiBit)) {
break;
}
lowPOT = 1U << hiBit;
if (lowPOT == (unsigned int)dataSize) {
return dataSize;
}
lowPOT = 1U << hiBit;
hiPOT = 1U << (hiBit + 1);
if (lowPOT == (unsigned int)dataSize) {
return dataSize;
}
hiPOT = 1U << (hiBit + 1);
if (hiPOT <= 1024) {
return hiPOT;
} else {
return iAlignUp(dataSize, 512);
}
if (hiPOT <= 1024) {
return hiPOT;
}
else {
return iAlignUp(dataSize, 512);
}
}
float getRand(void) { return (float)(rand() % 16); }
bool test0(void) {
float *h_Data, *h_Kernel, *h_ResultCPU, *h_ResultGPU;
bool test0(void)
{
float *h_Data, *h_Kernel, *h_ResultCPU, *h_ResultGPU;
float *d_Data, *d_PaddedData, *d_Kernel, *d_PaddedKernel;
float *d_Data, *d_PaddedData, *d_Kernel, *d_PaddedKernel;
fComplex *d_DataSpectrum, *d_KernelSpectrum;
fComplex *d_DataSpectrum, *d_KernelSpectrum;
cufftHandle fftPlanFwd, fftPlanInv;
cufftHandle fftPlanFwd, fftPlanInv;
bool bRetVal;
StopWatchInterface *hTimer = NULL;
sdkCreateTimer(&hTimer);
bool bRetVal;
StopWatchInterface *hTimer = NULL;
sdkCreateTimer(&hTimer);
printf("Testing built-in R2C / C2R FFT-based convolution\n");
const int kernelH = 7;
const int kernelW = 6;
const int kernelY = 3;
const int kernelX = 4;
const int dataH = 2000;
const int dataW = 2000;
const int fftH = snapTransformSize(dataH + kernelH - 1);
const int fftW = snapTransformSize(dataW + kernelW - 1);
printf("Testing built-in R2C / C2R FFT-based convolution\n");
const int kernelH = 7;
const int kernelW = 6;
const int kernelY = 3;
const int kernelX = 4;
const int dataH = 2000;
const int dataW = 2000;
const int fftH = snapTransformSize(dataH + kernelH - 1);
const int fftW = snapTransformSize(dataW + kernelW - 1);
printf("...allocating memory\n");
h_Data = (float *)malloc(dataH * dataW * sizeof(float));
h_Kernel = (float *)malloc(kernelH * kernelW * sizeof(float));
h_ResultCPU = (float *)malloc(dataH * dataW * sizeof(float));
h_ResultGPU = (float *)malloc(fftH * fftW * sizeof(float));
printf("...allocating memory\n");
h_Data = (float *)malloc(dataH * dataW * sizeof(float));
h_Kernel = (float *)malloc(kernelH * kernelW * sizeof(float));
h_ResultCPU = (float *)malloc(dataH * dataW * sizeof(float));
h_ResultGPU = (float *)malloc(fftH * fftW * sizeof(float));
checkCudaErrors(cudaMalloc((void **)&d_Data, dataH * dataW * sizeof(float)));
checkCudaErrors(
cudaMalloc((void **)&d_Kernel, kernelH * kernelW * sizeof(float)));
checkCudaErrors(cudaMalloc((void **)&d_Data, dataH * dataW * sizeof(float)));
checkCudaErrors(cudaMalloc((void **)&d_Kernel, kernelH * kernelW * sizeof(float)));
checkCudaErrors(
cudaMalloc((void **)&d_PaddedData, fftH * fftW * sizeof(float)));
checkCudaErrors(
cudaMalloc((void **)&d_PaddedKernel, fftH * fftW * sizeof(float)));
checkCudaErrors(cudaMalloc((void **)&d_PaddedData, fftH * fftW * sizeof(float)));
checkCudaErrors(cudaMalloc((void **)&d_PaddedKernel, fftH * fftW * sizeof(float)));
checkCudaErrors(cudaMalloc((void **)&d_DataSpectrum,
fftH * (fftW / 2 + 1) * sizeof(fComplex)));
checkCudaErrors(cudaMalloc((void **)&d_KernelSpectrum,
fftH * (fftW / 2 + 1) * sizeof(fComplex)));
checkCudaErrors(cudaMemset(d_KernelSpectrum, 0,
fftH * (fftW / 2 + 1) * sizeof(fComplex)));
checkCudaErrors(cudaMalloc((void **)&d_DataSpectrum, fftH * (fftW / 2 + 1) * sizeof(fComplex)));
checkCudaErrors(cudaMalloc((void **)&d_KernelSpectrum, fftH * (fftW / 2 + 1) * sizeof(fComplex)));
checkCudaErrors(cudaMemset(d_KernelSpectrum, 0, fftH * (fftW / 2 + 1) * sizeof(fComplex)));
printf("...generating random input data\n");
srand(2010);
printf("...generating random input data\n");
srand(2010);
for (int i = 0; i < dataH * dataW; i++) {
h_Data[i] = getRand();
}
for (int i = 0; i < kernelH * kernelW; i++) {
h_Kernel[i] = getRand();
}
printf("...creating R2C & C2R FFT plans for %i x %i\n", fftH, fftW);
checkCudaErrors(cufftPlan2d(&fftPlanFwd, fftH, fftW, CUFFT_R2C));
checkCudaErrors(cufftPlan2d(&fftPlanInv, fftH, fftW, CUFFT_C2R));
printf("...uploading to GPU and padding convolution kernel and input data\n");
checkCudaErrors(cudaMemcpy(d_Kernel, h_Kernel,
kernelH * kernelW * sizeof(float),
cudaMemcpyHostToDevice));
checkCudaErrors(cudaMemcpy(d_Data, h_Data, dataH * dataW * sizeof(float),
cudaMemcpyHostToDevice));
checkCudaErrors(cudaMemset(d_PaddedKernel, 0, fftH * fftW * sizeof(float)));
checkCudaErrors(cudaMemset(d_PaddedData, 0, fftH * fftW * sizeof(float)));
padKernel(d_PaddedKernel, d_Kernel, fftH, fftW, kernelH, kernelW, kernelY,
kernelX);
padDataClampToBorder(d_PaddedData, d_Data, fftH, fftW, dataH, dataW, kernelH,
kernelW, kernelY, kernelX);
// Not including kernel transformation into time measurement,
// since convolution kernel is not changed very frequently
printf("...transforming convolution kernel\n");
checkCudaErrors(cufftExecR2C(fftPlanFwd, (cufftReal *)d_PaddedKernel,
(cufftComplex *)d_KernelSpectrum));
printf("...running GPU FFT convolution: ");
checkCudaErrors(cudaDeviceSynchronize());
sdkResetTimer(&hTimer);
sdkStartTimer(&hTimer);
checkCudaErrors(cufftExecR2C(fftPlanFwd, (cufftReal *)d_PaddedData,
(cufftComplex *)d_DataSpectrum));
modulateAndNormalize(d_DataSpectrum, d_KernelSpectrum, fftH, fftW, 1);
checkCudaErrors(cufftExecC2R(fftPlanInv, (cufftComplex *)d_DataSpectrum,
(cufftReal *)d_PaddedData));
checkCudaErrors(cudaDeviceSynchronize());
sdkStopTimer(&hTimer);
double gpuTime = sdkGetTimerValue(&hTimer);
printf("%f MPix/s (%f ms)\n",
(double)dataH * (double)dataW * 1e-6 / (gpuTime * 0.001), gpuTime);
printf("...reading back GPU convolution results\n");
checkCudaErrors(cudaMemcpy(h_ResultGPU, d_PaddedData,
fftH * fftW * sizeof(float),
cudaMemcpyDeviceToHost));
printf("...running reference CPU convolution\n");
convolutionClampToBorderCPU(h_ResultCPU, h_Data, h_Kernel, dataH, dataW,
kernelH, kernelW, kernelY, kernelX);
printf("...comparing the results: ");
double sum_delta2 = 0;
double sum_ref2 = 0;
double max_delta_ref = 0;
for (int y = 0; y < dataH; y++)
for (int x = 0; x < dataW; x++) {
double rCPU = (double)h_ResultCPU[y * dataW + x];
double rGPU = (double)h_ResultGPU[y * fftW + x];
double delta = (rCPU - rGPU) * (rCPU - rGPU);
double ref = rCPU * rCPU + rCPU * rCPU;
if ((delta / ref) > max_delta_ref) {
max_delta_ref = delta / ref;
}
sum_delta2 += delta;
sum_ref2 += ref;
for (int i = 0; i < dataH * dataW; i++) {
h_Data[i] = getRand();
}
double L2norm = sqrt(sum_delta2 / sum_ref2);
printf("rel L2 = %E (max delta = %E)\n", L2norm, sqrt(max_delta_ref));
bRetVal = (L2norm < 1e-6) ? true : false;
printf(bRetVal ? "L2norm Error OK\n" : "L2norm Error too high!\n");
printf("...shutting down\n");
sdkDeleteTimer(&hTimer);
checkCudaErrors(cufftDestroy(fftPlanInv));
checkCudaErrors(cufftDestroy(fftPlanFwd));
checkCudaErrors(cudaFree(d_DataSpectrum));
checkCudaErrors(cudaFree(d_KernelSpectrum));
checkCudaErrors(cudaFree(d_PaddedData));
checkCudaErrors(cudaFree(d_PaddedKernel));
checkCudaErrors(cudaFree(d_Data));
checkCudaErrors(cudaFree(d_Kernel));
free(h_ResultGPU);
free(h_ResultCPU);
free(h_Data);
free(h_Kernel);
return bRetVal;
}
bool test1(void) {
float *h_Data, *h_Kernel, *h_ResultCPU, *h_ResultGPU;
float *d_Data, *d_Kernel, *d_PaddedData, *d_PaddedKernel;
fComplex *d_DataSpectrum0, *d_KernelSpectrum0, *d_DataSpectrum,
*d_KernelSpectrum;
cufftHandle fftPlan;
bool bRetVal;
StopWatchInterface *hTimer = NULL;
sdkCreateTimer(&hTimer);
printf("Testing custom R2C / C2R FFT-based convolution\n");
const uint fftPadding = 16;
const int kernelH = 7;
const int kernelW = 6;
const int kernelY = 3;
const int kernelX = 4;
const int dataH = 2000;
const int dataW = 2000;
const int fftH = snapTransformSize(dataH + kernelH - 1);
const int fftW = snapTransformSize(dataW + kernelW - 1);
printf("...allocating memory\n");
h_Data = (float *)malloc(dataH * dataW * sizeof(float));
h_Kernel = (float *)malloc(kernelH * kernelW * sizeof(float));
h_ResultCPU = (float *)malloc(dataH * dataW * sizeof(float));
h_ResultGPU = (float *)malloc(fftH * fftW * sizeof(float));
checkCudaErrors(cudaMalloc((void **)&d_Data, dataH * dataW * sizeof(float)));
checkCudaErrors(
cudaMalloc((void **)&d_Kernel, kernelH * kernelW * sizeof(float)));
checkCudaErrors(
cudaMalloc((void **)&d_PaddedData, fftH * fftW * sizeof(float)));
checkCudaErrors(
cudaMalloc((void **)&d_PaddedKernel, fftH * fftW * sizeof(float)));
checkCudaErrors(cudaMalloc((void **)&d_DataSpectrum0,
fftH * (fftW / 2) * sizeof(fComplex)));
checkCudaErrors(cudaMalloc((void **)&d_KernelSpectrum0,
fftH * (fftW / 2) * sizeof(fComplex)));
checkCudaErrors(
cudaMalloc((void **)&d_DataSpectrum,
fftH * (fftW / 2 + fftPadding) * sizeof(fComplex)));
checkCudaErrors(
cudaMalloc((void **)&d_KernelSpectrum,
fftH * (fftW / 2 + fftPadding) * sizeof(fComplex)));
printf("...generating random input data\n");
srand(2010);
for (int i = 0; i < dataH * dataW; i++) {
h_Data[i] = getRand();
}
for (int i = 0; i < kernelH * kernelW; i++) {
h_Kernel[i] = getRand();
}
printf("...creating C2C FFT plan for %i x %i\n", fftH, fftW / 2);
checkCudaErrors(cufftPlan2d(&fftPlan, fftH, fftW / 2, CUFFT_C2C));
printf("...uploading to GPU and padding convolution kernel and input data\n");
checkCudaErrors(cudaMemcpy(d_Data, h_Data, dataH * dataW * sizeof(float),
cudaMemcpyHostToDevice));
checkCudaErrors(cudaMemcpy(d_Kernel, h_Kernel,
kernelH * kernelW * sizeof(float),
cudaMemcpyHostToDevice));
checkCudaErrors(cudaMemset(d_PaddedData, 0, fftH * fftW * sizeof(float)));
checkCudaErrors(cudaMemset(d_PaddedKernel, 0, fftH * fftW * sizeof(float)));
padDataClampToBorder(d_PaddedData, d_Data, fftH, fftW, dataH, dataW, kernelH,
kernelW, kernelY, kernelX);
padKernel(d_PaddedKernel, d_Kernel, fftH, fftW, kernelH, kernelW, kernelY,
kernelX);
// CUFFT_INVERSE works just as well...
const int FFT_DIR = CUFFT_FORWARD;
// Not including kernel transformation into time measurement,
// since convolution kernel is not changed very frequently
printf("...transforming convolution kernel\n");
checkCudaErrors(cufftExecC2C(fftPlan, (cufftComplex *)d_PaddedKernel,
(cufftComplex *)d_KernelSpectrum0, FFT_DIR));
spPostprocess2D(d_KernelSpectrum, d_KernelSpectrum0, fftH, fftW / 2,
fftPadding, FFT_DIR);
printf("...running GPU FFT convolution: ");
checkCudaErrors(cudaDeviceSynchronize());
sdkResetTimer(&hTimer);
sdkStartTimer(&hTimer);
checkCudaErrors(cufftExecC2C(fftPlan, (cufftComplex *)d_PaddedData,
(cufftComplex *)d_DataSpectrum0, FFT_DIR));
spPostprocess2D(d_DataSpectrum, d_DataSpectrum0, fftH, fftW / 2, fftPadding,
FFT_DIR);
modulateAndNormalize(d_DataSpectrum, d_KernelSpectrum, fftH, fftW,
fftPadding);
spPreprocess2D(d_DataSpectrum0, d_DataSpectrum, fftH, fftW / 2, fftPadding,
-FFT_DIR);
checkCudaErrors(cufftExecC2C(fftPlan, (cufftComplex *)d_DataSpectrum0,
(cufftComplex *)d_PaddedData, -FFT_DIR));
checkCudaErrors(cudaDeviceSynchronize());
sdkStopTimer(&hTimer);
double gpuTime = sdkGetTimerValue(&hTimer);
printf("%f MPix/s (%f ms)\n",
(double)dataH * (double)dataW * 1e-6 / (gpuTime * 0.001), gpuTime);
printf("...reading back GPU FFT results\n");
checkCudaErrors(cudaMemcpy(h_ResultGPU, d_PaddedData,
fftH * fftW * sizeof(float),
cudaMemcpyDeviceToHost));
printf("...running reference CPU convolution\n");
convolutionClampToBorderCPU(h_ResultCPU, h_Data, h_Kernel, dataH, dataW,
kernelH, kernelW, kernelY, kernelX);
printf("...comparing the results: ");
double sum_delta2 = 0;
double sum_ref2 = 0;
double max_delta_ref = 0;
for (int y = 0; y < dataH; y++)
for (int x = 0; x < dataW; x++) {
double rCPU = (double)h_ResultCPU[y * dataW + x];
double rGPU = (double)h_ResultGPU[y * fftW + x];
double delta = (rCPU - rGPU) * (rCPU - rGPU);
double ref = rCPU * rCPU + rCPU * rCPU;
if ((delta / ref) > max_delta_ref) {
max_delta_ref = delta / ref;
}
sum_delta2 += delta;
sum_ref2 += ref;
for (int i = 0; i < kernelH * kernelW; i++) {
h_Kernel[i] = getRand();
}
double L2norm = sqrt(sum_delta2 / sum_ref2);
printf("rel L2 = %E (max delta = %E)\n", L2norm, sqrt(max_delta_ref));
bRetVal = (L2norm < 1e-6) ? true : false;
printf(bRetVal ? "L2norm Error OK\n" : "L2norm Error too high!\n");
printf("...creating R2C & C2R FFT plans for %i x %i\n", fftH, fftW);
checkCudaErrors(cufftPlan2d(&fftPlanFwd, fftH, fftW, CUFFT_R2C));
checkCudaErrors(cufftPlan2d(&fftPlanInv, fftH, fftW, CUFFT_C2R));
printf("...shutting down\n");
sdkDeleteTimer(&hTimer);
checkCudaErrors(cufftDestroy(fftPlan));
printf("...uploading to GPU and padding convolution kernel and input data\n");
checkCudaErrors(cudaMemcpy(d_Kernel, h_Kernel, kernelH * kernelW * sizeof(float), cudaMemcpyHostToDevice));
checkCudaErrors(cudaMemcpy(d_Data, h_Data, dataH * dataW * sizeof(float), cudaMemcpyHostToDevice));
checkCudaErrors(cudaMemset(d_PaddedKernel, 0, fftH * fftW * sizeof(float)));
checkCudaErrors(cudaMemset(d_PaddedData, 0, fftH * fftW * sizeof(float)));
checkCudaErrors(cudaFree(d_KernelSpectrum));
checkCudaErrors(cudaFree(d_DataSpectrum));
checkCudaErrors(cudaFree(d_KernelSpectrum0));
checkCudaErrors(cudaFree(d_DataSpectrum0));
checkCudaErrors(cudaFree(d_PaddedKernel));
checkCudaErrors(cudaFree(d_PaddedData));
checkCudaErrors(cudaFree(d_Kernel));
checkCudaErrors(cudaFree(d_Data));
padKernel(d_PaddedKernel, d_Kernel, fftH, fftW, kernelH, kernelW, kernelY, kernelX);
free(h_ResultGPU);
free(h_ResultCPU);
free(h_Kernel);
free(h_Data);
padDataClampToBorder(d_PaddedData, d_Data, fftH, fftW, dataH, dataW, kernelH, kernelW, kernelY, kernelX);
return bRetVal;
// Not including kernel transformation into time measurement,
// since convolution kernel is not changed very frequently
printf("...transforming convolution kernel\n");
checkCudaErrors(cufftExecR2C(fftPlanFwd, (cufftReal *)d_PaddedKernel, (cufftComplex *)d_KernelSpectrum));
printf("...running GPU FFT convolution: ");
checkCudaErrors(cudaDeviceSynchronize());
sdkResetTimer(&hTimer);
sdkStartTimer(&hTimer);
checkCudaErrors(cufftExecR2C(fftPlanFwd, (cufftReal *)d_PaddedData, (cufftComplex *)d_DataSpectrum));
modulateAndNormalize(d_DataSpectrum, d_KernelSpectrum, fftH, fftW, 1);
checkCudaErrors(cufftExecC2R(fftPlanInv, (cufftComplex *)d_DataSpectrum, (cufftReal *)d_PaddedData));
checkCudaErrors(cudaDeviceSynchronize());
sdkStopTimer(&hTimer);
double gpuTime = sdkGetTimerValue(&hTimer);
printf("%f MPix/s (%f ms)\n", (double)dataH * (double)dataW * 1e-6 / (gpuTime * 0.001), gpuTime);
printf("...reading back GPU convolution results\n");
checkCudaErrors(cudaMemcpy(h_ResultGPU, d_PaddedData, fftH * fftW * sizeof(float), cudaMemcpyDeviceToHost));
printf("...running reference CPU convolution\n");
convolutionClampToBorderCPU(h_ResultCPU, h_Data, h_Kernel, dataH, dataW, kernelH, kernelW, kernelY, kernelX);
printf("...comparing the results: ");
double sum_delta2 = 0;
double sum_ref2 = 0;
double max_delta_ref = 0;
for (int y = 0; y < dataH; y++)
for (int x = 0; x < dataW; x++) {
double rCPU = (double)h_ResultCPU[y * dataW + x];
double rGPU = (double)h_ResultGPU[y * fftW + x];
double delta = (rCPU - rGPU) * (rCPU - rGPU);
double ref = rCPU * rCPU + rCPU * rCPU;
if ((delta / ref) > max_delta_ref) {
max_delta_ref = delta / ref;
}
sum_delta2 += delta;
sum_ref2 += ref;
}
double L2norm = sqrt(sum_delta2 / sum_ref2);
printf("rel L2 = %E (max delta = %E)\n", L2norm, sqrt(max_delta_ref));
bRetVal = (L2norm < 1e-6) ? true : false;
printf(bRetVal ? "L2norm Error OK\n" : "L2norm Error too high!\n");
printf("...shutting down\n");
sdkDeleteTimer(&hTimer);
checkCudaErrors(cufftDestroy(fftPlanInv));
checkCudaErrors(cufftDestroy(fftPlanFwd));
checkCudaErrors(cudaFree(d_DataSpectrum));
checkCudaErrors(cudaFree(d_KernelSpectrum));
checkCudaErrors(cudaFree(d_PaddedData));
checkCudaErrors(cudaFree(d_PaddedKernel));
checkCudaErrors(cudaFree(d_Data));
checkCudaErrors(cudaFree(d_Kernel));
free(h_ResultGPU);
free(h_ResultCPU);
free(h_Data);
free(h_Kernel);
return bRetVal;
}
bool test2(void) {
float *h_Data, *h_Kernel, *h_ResultCPU, *h_ResultGPU;
bool test1(void)
{
float *h_Data, *h_Kernel, *h_ResultCPU, *h_ResultGPU;
float *d_Data, *d_Kernel, *d_PaddedData, *d_PaddedKernel;
float *d_Data, *d_Kernel, *d_PaddedData, *d_PaddedKernel;
fComplex *d_DataSpectrum0, *d_KernelSpectrum0;
fComplex *d_DataSpectrum0, *d_KernelSpectrum0, *d_DataSpectrum, *d_KernelSpectrum;
cufftHandle fftPlan;
cufftHandle fftPlan;
bool bRetVal;
StopWatchInterface *hTimer = NULL;
sdkCreateTimer(&hTimer);
bool bRetVal;
StopWatchInterface *hTimer = NULL;
sdkCreateTimer(&hTimer);
printf("Testing updated custom R2C / C2R FFT-based convolution\n");
const int kernelH = 7;
const int kernelW = 6;
const int kernelY = 3;
const int kernelX = 4;
const int dataH = 2000;
const int dataW = 2000;
const int fftH = snapTransformSize(dataH + kernelH - 1);
const int fftW = snapTransformSize(dataW + kernelW - 1);
printf("Testing custom R2C / C2R FFT-based convolution\n");
const uint fftPadding = 16;
const int kernelH = 7;
const int kernelW = 6;
const int kernelY = 3;
const int kernelX = 4;
const int dataH = 2000;
const int dataW = 2000;
const int fftH = snapTransformSize(dataH + kernelH - 1);
const int fftW = snapTransformSize(dataW + kernelW - 1);
printf("...allocating memory\n");
h_Data = (float *)malloc(dataH * dataW * sizeof(float));
h_Kernel = (float *)malloc(kernelH * kernelW * sizeof(float));
h_ResultCPU = (float *)malloc(dataH * dataW * sizeof(float));
h_ResultGPU = (float *)malloc(fftH * fftW * sizeof(float));
printf("...allocating memory\n");
h_Data = (float *)malloc(dataH * dataW * sizeof(float));
h_Kernel = (float *)malloc(kernelH * kernelW * sizeof(float));
h_ResultCPU = (float *)malloc(dataH * dataW * sizeof(float));
h_ResultGPU = (float *)malloc(fftH * fftW * sizeof(float));
checkCudaErrors(cudaMalloc((void **)&d_Data, dataH * dataW * sizeof(float)));
checkCudaErrors(
cudaMalloc((void **)&d_Kernel, kernelH * kernelW * sizeof(float)));
checkCudaErrors(cudaMalloc((void **)&d_Data, dataH * dataW * sizeof(float)));
checkCudaErrors(cudaMalloc((void **)&d_Kernel, kernelH * kernelW * sizeof(float)));
checkCudaErrors(
cudaMalloc((void **)&d_PaddedData, fftH * fftW * sizeof(float)));
checkCudaErrors(
cudaMalloc((void **)&d_PaddedKernel, fftH * fftW * sizeof(float)));
checkCudaErrors(cudaMalloc((void **)&d_PaddedData, fftH * fftW * sizeof(float)));
checkCudaErrors(cudaMalloc((void **)&d_PaddedKernel, fftH * fftW * sizeof(float)));
checkCudaErrors(cudaMalloc((void **)&d_DataSpectrum0,
fftH * (fftW / 2) * sizeof(fComplex)));
checkCudaErrors(cudaMalloc((void **)&d_KernelSpectrum0,
fftH * (fftW / 2) * sizeof(fComplex)));
checkCudaErrors(cudaMalloc((void **)&d_DataSpectrum0, fftH * (fftW / 2) * sizeof(fComplex)));
checkCudaErrors(cudaMalloc((void **)&d_KernelSpectrum0, fftH * (fftW / 2) * sizeof(fComplex)));
checkCudaErrors(cudaMalloc((void **)&d_DataSpectrum, fftH * (fftW / 2 + fftPadding) * sizeof(fComplex)));
checkCudaErrors(cudaMalloc((void **)&d_KernelSpectrum, fftH * (fftW / 2 + fftPadding) * sizeof(fComplex)));
printf("...generating random input data\n");
srand(2010);
printf("...generating random input data\n");
srand(2010);
for (int i = 0; i < dataH * dataW; i++) {
h_Data[i] = getRand();
}
for (int i = 0; i < kernelH * kernelW; i++) {
h_Kernel[i] = getRand();
}
printf("...creating C2C FFT plan for %i x %i\n", fftH, fftW / 2);
checkCudaErrors(cufftPlan2d(&fftPlan, fftH, fftW / 2, CUFFT_C2C));
printf("...uploading to GPU and padding convolution kernel and input data\n");
checkCudaErrors(cudaMemcpy(d_Data, h_Data, dataH * dataW * sizeof(float),
cudaMemcpyHostToDevice));
checkCudaErrors(cudaMemcpy(d_Kernel, h_Kernel,
kernelH * kernelW * sizeof(float),
cudaMemcpyHostToDevice));
checkCudaErrors(cudaMemset(d_PaddedData, 0, fftH * fftW * sizeof(float)));
checkCudaErrors(cudaMemset(d_PaddedKernel, 0, fftH * fftW * sizeof(float)));
padDataClampToBorder(d_PaddedData, d_Data, fftH, fftW, dataH, dataW, kernelH,
kernelW, kernelY, kernelX);
padKernel(d_PaddedKernel, d_Kernel, fftH, fftW, kernelH, kernelW, kernelY,
kernelX);
// CUFFT_INVERSE works just as well...
const int FFT_DIR = CUFFT_FORWARD;
// Not including kernel transformation into time measurement,
// since convolution kernel is not changed very frequently
printf("...transforming convolution kernel\n");
checkCudaErrors(cufftExecC2C(fftPlan, (cufftComplex *)d_PaddedKernel,
(cufftComplex *)d_KernelSpectrum0, FFT_DIR));
printf("...running GPU FFT convolution: ");
checkCudaErrors(cudaDeviceSynchronize());
sdkResetTimer(&hTimer);
sdkStartTimer(&hTimer);
checkCudaErrors(cufftExecC2C(fftPlan, (cufftComplex *)d_PaddedData,
(cufftComplex *)d_DataSpectrum0, FFT_DIR));
spProcess2D(d_DataSpectrum0, d_DataSpectrum0, d_KernelSpectrum0, fftH,
fftW / 2, FFT_DIR);
checkCudaErrors(cufftExecC2C(fftPlan, (cufftComplex *)d_DataSpectrum0,
(cufftComplex *)d_PaddedData, -FFT_DIR));
checkCudaErrors(cudaDeviceSynchronize());
sdkStopTimer(&hTimer);
double gpuTime = sdkGetTimerValue(&hTimer);
printf("%f MPix/s (%f ms)\n",
(double)dataH * (double)dataW * 1e-6 / (gpuTime * 0.001), gpuTime);
printf("...reading back GPU FFT results\n");
checkCudaErrors(cudaMemcpy(h_ResultGPU, d_PaddedData,
fftH * fftW * sizeof(float),
cudaMemcpyDeviceToHost));
printf("...running reference CPU convolution\n");
convolutionClampToBorderCPU(h_ResultCPU, h_Data, h_Kernel, dataH, dataW,
kernelH, kernelW, kernelY, kernelX);
printf("...comparing the results: ");
double sum_delta2 = 0;
double sum_ref2 = 0;
double max_delta_ref = 0;
for (int y = 0; y < dataH; y++) {
for (int x = 0; x < dataW; x++) {
double rCPU = (double)h_ResultCPU[y * dataW + x];
double rGPU = (double)h_ResultGPU[y * fftW + x];
double delta = (rCPU - rGPU) * (rCPU - rGPU);
double ref = rCPU * rCPU + rCPU * rCPU;
if ((delta / ref) > max_delta_ref) {
max_delta_ref = delta / ref;
}
sum_delta2 += delta;
sum_ref2 += ref;
for (int i = 0; i < dataH * dataW; i++) {
h_Data[i] = getRand();
}
}
double L2norm = sqrt(sum_delta2 / sum_ref2);
printf("rel L2 = %E (max delta = %E)\n", L2norm, sqrt(max_delta_ref));
bRetVal = (L2norm < 1e-6) ? true : false;
printf(bRetVal ? "L2norm Error OK\n" : "L2norm Error too high!\n");
for (int i = 0; i < kernelH * kernelW; i++) {
h_Kernel[i] = getRand();
}
printf("...shutting down\n");
sdkDeleteTimer(&hTimer);
checkCudaErrors(cufftDestroy(fftPlan));
printf("...creating C2C FFT plan for %i x %i\n", fftH, fftW / 2);
checkCudaErrors(cufftPlan2d(&fftPlan, fftH, fftW / 2, CUFFT_C2C));
checkCudaErrors(cudaFree(d_KernelSpectrum0));
checkCudaErrors(cudaFree(d_DataSpectrum0));
checkCudaErrors(cudaFree(d_PaddedKernel));
checkCudaErrors(cudaFree(d_PaddedData));
checkCudaErrors(cudaFree(d_Kernel));
checkCudaErrors(cudaFree(d_Data));
printf("...uploading to GPU and padding convolution kernel and input data\n");
checkCudaErrors(cudaMemcpy(d_Data, h_Data, dataH * dataW * sizeof(float), cudaMemcpyHostToDevice));
checkCudaErrors(cudaMemcpy(d_Kernel, h_Kernel, kernelH * kernelW * sizeof(float), cudaMemcpyHostToDevice));
checkCudaErrors(cudaMemset(d_PaddedData, 0, fftH * fftW * sizeof(float)));
checkCudaErrors(cudaMemset(d_PaddedKernel, 0, fftH * fftW * sizeof(float)));
free(h_ResultGPU);
free(h_ResultCPU);
free(h_Kernel);
free(h_Data);
padDataClampToBorder(d_PaddedData, d_Data, fftH, fftW, dataH, dataW, kernelH, kernelW, kernelY, kernelX);
return bRetVal;
padKernel(d_PaddedKernel, d_Kernel, fftH, fftW, kernelH, kernelW, kernelY, kernelX);
// CUFFT_INVERSE works just as well...
const int FFT_DIR = CUFFT_FORWARD;
// Not including kernel transformation into time measurement,
// since convolution kernel is not changed very frequently
printf("...transforming convolution kernel\n");
checkCudaErrors(cufftExecC2C(fftPlan, (cufftComplex *)d_PaddedKernel, (cufftComplex *)d_KernelSpectrum0, FFT_DIR));
spPostprocess2D(d_KernelSpectrum, d_KernelSpectrum0, fftH, fftW / 2, fftPadding, FFT_DIR);
printf("...running GPU FFT convolution: ");
checkCudaErrors(cudaDeviceSynchronize());
sdkResetTimer(&hTimer);
sdkStartTimer(&hTimer);
checkCudaErrors(cufftExecC2C(fftPlan, (cufftComplex *)d_PaddedData, (cufftComplex *)d_DataSpectrum0, FFT_DIR));
spPostprocess2D(d_DataSpectrum, d_DataSpectrum0, fftH, fftW / 2, fftPadding, FFT_DIR);
modulateAndNormalize(d_DataSpectrum, d_KernelSpectrum, fftH, fftW, fftPadding);
spPreprocess2D(d_DataSpectrum0, d_DataSpectrum, fftH, fftW / 2, fftPadding, -FFT_DIR);
checkCudaErrors(cufftExecC2C(fftPlan, (cufftComplex *)d_DataSpectrum0, (cufftComplex *)d_PaddedData, -FFT_DIR));
checkCudaErrors(cudaDeviceSynchronize());
sdkStopTimer(&hTimer);
double gpuTime = sdkGetTimerValue(&hTimer);
printf("%f MPix/s (%f ms)\n", (double)dataH * (double)dataW * 1e-6 / (gpuTime * 0.001), gpuTime);
printf("...reading back GPU FFT results\n");
checkCudaErrors(cudaMemcpy(h_ResultGPU, d_PaddedData, fftH * fftW * sizeof(float), cudaMemcpyDeviceToHost));
printf("...running reference CPU convolution\n");
convolutionClampToBorderCPU(h_ResultCPU, h_Data, h_Kernel, dataH, dataW, kernelH, kernelW, kernelY, kernelX);
printf("...comparing the results: ");
double sum_delta2 = 0;
double sum_ref2 = 0;
double max_delta_ref = 0;
for (int y = 0; y < dataH; y++)
for (int x = 0; x < dataW; x++) {
double rCPU = (double)h_ResultCPU[y * dataW + x];
double rGPU = (double)h_ResultGPU[y * fftW + x];
double delta = (rCPU - rGPU) * (rCPU - rGPU);
double ref = rCPU * rCPU + rCPU * rCPU;
if ((delta / ref) > max_delta_ref) {
max_delta_ref = delta / ref;
}
sum_delta2 += delta;
sum_ref2 += ref;
}
double L2norm = sqrt(sum_delta2 / sum_ref2);
printf("rel L2 = %E (max delta = %E)\n", L2norm, sqrt(max_delta_ref));
bRetVal = (L2norm < 1e-6) ? true : false;
printf(bRetVal ? "L2norm Error OK\n" : "L2norm Error too high!\n");
printf("...shutting down\n");
sdkDeleteTimer(&hTimer);
checkCudaErrors(cufftDestroy(fftPlan));
checkCudaErrors(cudaFree(d_KernelSpectrum));
checkCudaErrors(cudaFree(d_DataSpectrum));
checkCudaErrors(cudaFree(d_KernelSpectrum0));
checkCudaErrors(cudaFree(d_DataSpectrum0));
checkCudaErrors(cudaFree(d_PaddedKernel));
checkCudaErrors(cudaFree(d_PaddedData));
checkCudaErrors(cudaFree(d_Kernel));
checkCudaErrors(cudaFree(d_Data));
free(h_ResultGPU);
free(h_ResultCPU);
free(h_Kernel);
free(h_Data);
return bRetVal;
}
int main(int argc, char **argv) {
printf("[%s] - Starting...\n", argv[0]);
bool test2(void)
{
float *h_Data, *h_Kernel, *h_ResultCPU, *h_ResultGPU;
// Use command-line specified CUDA device, otherwise use device with highest
// Gflops/s
findCudaDevice(argc, (const char **)argv);
float *d_Data, *d_Kernel, *d_PaddedData, *d_PaddedKernel;
int nFailures = 0;
fComplex *d_DataSpectrum0, *d_KernelSpectrum0;
if (!test0()) {
nFailures++;
}
cufftHandle fftPlan;
if (!test1()) {
nFailures++;
}
bool bRetVal;
StopWatchInterface *hTimer = NULL;
sdkCreateTimer(&hTimer);
if (!test2()) {
nFailures++;
}
printf("Testing updated custom R2C / C2R FFT-based convolution\n");
const int kernelH = 7;
const int kernelW = 6;
const int kernelY = 3;
const int kernelX = 4;
const int dataH = 2000;
const int dataW = 2000;
const int fftH = snapTransformSize(dataH + kernelH - 1);
const int fftW = snapTransformSize(dataW + kernelW - 1);
printf("Test Summary: %d errors\n", nFailures);
printf("...allocating memory\n");
h_Data = (float *)malloc(dataH * dataW * sizeof(float));
h_Kernel = (float *)malloc(kernelH * kernelW * sizeof(float));
h_ResultCPU = (float *)malloc(dataH * dataW * sizeof(float));
h_ResultGPU = (float *)malloc(fftH * fftW * sizeof(float));
if (nFailures > 0) {
printf("Test failed!\n");
exit(EXIT_FAILURE);
}
checkCudaErrors(cudaMalloc((void **)&d_Data, dataH * dataW * sizeof(float)));
checkCudaErrors(cudaMalloc((void **)&d_Kernel, kernelH * kernelW * sizeof(float)));
printf("Test passed\n");
exit(EXIT_SUCCESS);
checkCudaErrors(cudaMalloc((void **)&d_PaddedData, fftH * fftW * sizeof(float)));
checkCudaErrors(cudaMalloc((void **)&d_PaddedKernel, fftH * fftW * sizeof(float)));
checkCudaErrors(cudaMalloc((void **)&d_DataSpectrum0, fftH * (fftW / 2) * sizeof(fComplex)));
checkCudaErrors(cudaMalloc((void **)&d_KernelSpectrum0, fftH * (fftW / 2) * sizeof(fComplex)));
printf("...generating random input data\n");
srand(2010);
for (int i = 0; i < dataH * dataW; i++) {
h_Data[i] = getRand();
}
for (int i = 0; i < kernelH * kernelW; i++) {
h_Kernel[i] = getRand();
}
printf("...creating C2C FFT plan for %i x %i\n", fftH, fftW / 2);
checkCudaErrors(cufftPlan2d(&fftPlan, fftH, fftW / 2, CUFFT_C2C));
printf("...uploading to GPU and padding convolution kernel and input data\n");
checkCudaErrors(cudaMemcpy(d_Data, h_Data, dataH * dataW * sizeof(float), cudaMemcpyHostToDevice));
checkCudaErrors(cudaMemcpy(d_Kernel, h_Kernel, kernelH * kernelW * sizeof(float), cudaMemcpyHostToDevice));
checkCudaErrors(cudaMemset(d_PaddedData, 0, fftH * fftW * sizeof(float)));
checkCudaErrors(cudaMemset(d_PaddedKernel, 0, fftH * fftW * sizeof(float)));
padDataClampToBorder(d_PaddedData, d_Data, fftH, fftW, dataH, dataW, kernelH, kernelW, kernelY, kernelX);
padKernel(d_PaddedKernel, d_Kernel, fftH, fftW, kernelH, kernelW, kernelY, kernelX);
// CUFFT_INVERSE works just as well...
const int FFT_DIR = CUFFT_FORWARD;
// Not including kernel transformation into time measurement,
// since convolution kernel is not changed very frequently
printf("...transforming convolution kernel\n");
checkCudaErrors(cufftExecC2C(fftPlan, (cufftComplex *)d_PaddedKernel, (cufftComplex *)d_KernelSpectrum0, FFT_DIR));
printf("...running GPU FFT convolution: ");
checkCudaErrors(cudaDeviceSynchronize());
sdkResetTimer(&hTimer);
sdkStartTimer(&hTimer);
checkCudaErrors(cufftExecC2C(fftPlan, (cufftComplex *)d_PaddedData, (cufftComplex *)d_DataSpectrum0, FFT_DIR));
spProcess2D(d_DataSpectrum0, d_DataSpectrum0, d_KernelSpectrum0, fftH, fftW / 2, FFT_DIR);
checkCudaErrors(cufftExecC2C(fftPlan, (cufftComplex *)d_DataSpectrum0, (cufftComplex *)d_PaddedData, -FFT_DIR));
checkCudaErrors(cudaDeviceSynchronize());
sdkStopTimer(&hTimer);
double gpuTime = sdkGetTimerValue(&hTimer);
printf("%f MPix/s (%f ms)\n", (double)dataH * (double)dataW * 1e-6 / (gpuTime * 0.001), gpuTime);
printf("...reading back GPU FFT results\n");
checkCudaErrors(cudaMemcpy(h_ResultGPU, d_PaddedData, fftH * fftW * sizeof(float), cudaMemcpyDeviceToHost));
printf("...running reference CPU convolution\n");
convolutionClampToBorderCPU(h_ResultCPU, h_Data, h_Kernel, dataH, dataW, kernelH, kernelW, kernelY, kernelX);
printf("...comparing the results: ");
double sum_delta2 = 0;
double sum_ref2 = 0;
double max_delta_ref = 0;
for (int y = 0; y < dataH; y++) {
for (int x = 0; x < dataW; x++) {
double rCPU = (double)h_ResultCPU[y * dataW + x];
double rGPU = (double)h_ResultGPU[y * fftW + x];
double delta = (rCPU - rGPU) * (rCPU - rGPU);
double ref = rCPU * rCPU + rCPU * rCPU;
if ((delta / ref) > max_delta_ref) {
max_delta_ref = delta / ref;
}
sum_delta2 += delta;
sum_ref2 += ref;
}
}
double L2norm = sqrt(sum_delta2 / sum_ref2);
printf("rel L2 = %E (max delta = %E)\n", L2norm, sqrt(max_delta_ref));
bRetVal = (L2norm < 1e-6) ? true : false;
printf(bRetVal ? "L2norm Error OK\n" : "L2norm Error too high!\n");
printf("...shutting down\n");
sdkDeleteTimer(&hTimer);
checkCudaErrors(cufftDestroy(fftPlan));
checkCudaErrors(cudaFree(d_KernelSpectrum0));
checkCudaErrors(cudaFree(d_DataSpectrum0));
checkCudaErrors(cudaFree(d_PaddedKernel));
checkCudaErrors(cudaFree(d_PaddedData));
checkCudaErrors(cudaFree(d_Kernel));
checkCudaErrors(cudaFree(d_Data));
free(h_ResultGPU);
free(h_ResultCPU);
free(h_Kernel);
free(h_Data);
return bRetVal;
}
int main(int argc, char **argv)
{
printf("[%s] - Starting...\n", argv[0]);
// Use command-line specified CUDA device, otherwise use device with highest
// Gflops/s
findCudaDevice(argc, (const char **)argv);
int nFailures = 0;
if (!test0()) {
nFailures++;
}
if (!test1()) {
nFailures++;
}
if (!test2()) {
nFailures++;
}
printf("Test Summary: %d errors\n", nFailures);
if (nFailures > 0) {
printf("Test failed!\n");
exit(EXIT_FAILURE);
}
printf("Test passed\n");
exit(EXIT_SUCCESS);
}