mirror of
https://github.com/NVIDIA/cuda-samples.git
synced 2026-10-26 06:38:16 +08:00
Apply consistent code formatting across the repo. Add clang-format and pre-commit hooks.
This commit is contained in:
Rob Armstrong
@@ -59,18 +59,18 @@
|
||||
// but carefully enough to avoid local memory use.
|
||||
|
||||
#include <assert.h>
|
||||
#include <stdio.h>
|
||||
#include <cuda.h>
|
||||
#include <mma.h>
|
||||
#include <cooperative_groups.h>
|
||||
#include <cooperative_groups/memcpy_async.h>
|
||||
#include <cuda/std/type_traits>
|
||||
#include <cuda.h>
|
||||
#include <cuda/barrier>
|
||||
#include <cuda/pipeline>
|
||||
#include <cuda/std/type_traits>
|
||||
#include <mma.h>
|
||||
#include <stdio.h>
|
||||
|
||||
// helper functions and utilities to work with CUDA
|
||||
#include <helper_functions.h>
|
||||
#include <helper_cuda.h>
|
||||
#include <helper_functions.h>
|
||||
|
||||
// Externally configurable parameters.
|
||||
|
||||
@@ -111,7 +111,7 @@
|
||||
|
||||
// Implementation constants.
|
||||
|
||||
#define WARPS_PER_BLOCK 8
|
||||
#define WARPS_PER_BLOCK 8
|
||||
#define THREADS_PER_BLOCK (WARP_SIZE * WARPS_PER_BLOCK)
|
||||
|
||||
#if SHARED_MEMORY_LIMIT_64K
|
||||
@@ -127,10 +127,10 @@
|
||||
#define CHUNK_K 16
|
||||
#endif
|
||||
|
||||
#define CHUNK_LINE_BYTES (CHUNK_K * K * sizeof(double))
|
||||
#define WARP_COPY_BYTES (WARP_SIZE * sizeof(int4))
|
||||
#define CHUNK_LINE_BYTES (CHUNK_K * K * sizeof(double))
|
||||
#define WARP_COPY_BYTES (WARP_SIZE * sizeof(int4))
|
||||
#define CHUNK_COPY_LINES_PER_WARP (WARP_COPY_BYTES / CHUNK_LINE_BYTES)
|
||||
#define CHUNK_COPY_LINE_LANES (WARP_SIZE / CHUNK_COPY_LINES_PER_WARP)
|
||||
#define CHUNK_COPY_LINE_LANES (WARP_SIZE / CHUNK_COPY_LINES_PER_WARP)
|
||||
|
||||
#define BLOCK_ROW_WARPS 2
|
||||
#define BLOCK_COL_WARPS 4
|
||||
@@ -160,26 +160,28 @@
|
||||
// we must keep each row and column 256-bit aligned, as required by nvcuda::wmma::load_matrix_sync.
|
||||
#define SKEW_DOUBLE 4
|
||||
|
||||
#define checkKernelErrors(expr) do { \
|
||||
expr; \
|
||||
\
|
||||
cudaError_t __err = cudaGetLastError(); \
|
||||
if (__err != cudaSuccess) { \
|
||||
printf("Line %d: '%s' failed: %s\n", __LINE__, # expr, cudaGetErrorString(__err)); \
|
||||
abort(); \
|
||||
} \
|
||||
} while(0)
|
||||
#define checkKernelErrors(expr) \
|
||||
do { \
|
||||
expr; \
|
||||
\
|
||||
cudaError_t __err = cudaGetLastError(); \
|
||||
if (__err != cudaSuccess) { \
|
||||
printf("Line %d: '%s' failed: %s\n", __LINE__, #expr, cudaGetErrorString(__err)); \
|
||||
abort(); \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
enum kernels
|
||||
{
|
||||
dmma_shmem_gemm_async_copy = 0, // DMMA shmem using kernel with async_copy
|
||||
dmma_shmem_gemm_cg_async_copy = 1, // DMMA shmem using kernel with cooperative groups async_copy
|
||||
dmma_shmem_gemm = 2, // DMMA shmem using kernel normal copy (without async_copy).
|
||||
simple_dmma_gemm = 3 // DMMA non-shmem using simple kernel.
|
||||
enum kernels {
|
||||
dmma_shmem_gemm_async_copy = 0, // DMMA shmem using kernel with async_copy
|
||||
dmma_shmem_gemm_cg_async_copy = 1, // DMMA shmem using kernel with cooperative groups async_copy
|
||||
dmma_shmem_gemm = 2, // DMMA shmem using kernel normal copy (without async_copy).
|
||||
simple_dmma_gemm = 3 // DMMA non-shmem using simple kernel.
|
||||
};
|
||||
|
||||
const char* kernelNames[] = {"compute_dgemm_async_copy", "compute_dgemm_cg_async_copy",
|
||||
"compute_dgemm", "simple_wmma_gemm"};
|
||||
const char *kernelNames[] = {"compute_dgemm_async_copy",
|
||||
"compute_dgemm_cg_async_copy",
|
||||
"compute_dgemm",
|
||||
"simple_wmma_gemm"};
|
||||
|
||||
using namespace nvcuda;
|
||||
namespace cg = cooperative_groups;
|
||||
@@ -188,18 +190,18 @@ __host__ void init_host_matrices(double *a, double *b, double *c)
|
||||
{
|
||||
for (int i = 0; i < M_GLOBAL; i++) {
|
||||
for (int j = 0; j < K_GLOBAL; j++) {
|
||||
a[i*K_GLOBAL+j] = (double) (rand() % 3);
|
||||
a[i * K_GLOBAL + j] = (double)(rand() % 3);
|
||||
}
|
||||
}
|
||||
|
||||
for (int i = 0; i < N_GLOBAL; i++) {
|
||||
for (int j = 0; j < K_GLOBAL; j++) {
|
||||
b[i*K_GLOBAL+j] = (double) (rand() % 3);
|
||||
b[i * K_GLOBAL + j] = (double)(rand() % 3);
|
||||
}
|
||||
}
|
||||
|
||||
for (int t = 0; t < M_GLOBAL * N_GLOBAL; t++) {
|
||||
c[t] = (double) (rand() % 3);
|
||||
c[t] = (double)(rand() % 3);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -217,10 +219,12 @@ __global__ void compute_dgemm(const double *A, const double *B, const double *C,
|
||||
|
||||
|
||||
// This pointer is used to access the C and D matrix tiles this warp computes.
|
||||
double *shmem_warp_tile_ptr = (double*)&shmem[0][0] + (warpId / BLOCK_ROW_WARPS) * SHMEM_STRIDE * N * BLOCK_ROW_WARPS + (warpId % BLOCK_ROW_WARPS) * SHMEM_OFFSET;
|
||||
double *shmem_warp_tile_ptr = (double *)&shmem[0][0]
|
||||
+ (warpId / BLOCK_ROW_WARPS) * SHMEM_STRIDE * N * BLOCK_ROW_WARPS
|
||||
+ (warpId % BLOCK_ROW_WARPS) * SHMEM_OFFSET;
|
||||
|
||||
// This pointer is used to stream the C and D matrices block-wide tile to and from shared memory.
|
||||
double *shmem_warp_stream_ptr = (double*)&shmem[0][0] + warpId * SHMEM_STRIDE * N;
|
||||
double *shmem_warp_stream_ptr = (double *)&shmem[0][0] + warpId * SHMEM_STRIDE * N;
|
||||
|
||||
// Adjust the beta scaler, as it'll be multiplied by alpha at the end of
|
||||
// each tile computation. Technically this is not generally correct (may result
|
||||
@@ -230,7 +234,7 @@ __global__ void compute_dgemm(const double *A, const double *B, const double *C,
|
||||
// Each CTA slides along the 64 x 64 tiles from the top left corner of the matrix to the
|
||||
// right and down, and selects the next tile to compute. Once there's no such tile,
|
||||
// all warps in this CTA exit.
|
||||
for(unsigned int block_pos = blockIdx.x;; block_pos += gridDim.x) {
|
||||
for (unsigned int block_pos = blockIdx.x;; block_pos += gridDim.x) {
|
||||
const unsigned int block_tile_i = ((block_pos * BLOCK_ROW_TILES) / N_TILES) * (BLOCK_COL_TILES);
|
||||
const unsigned int block_tile_j = (block_pos * BLOCK_COL_TILES) % N_TILES;
|
||||
|
||||
@@ -240,7 +244,7 @@ __global__ void compute_dgemm(const double *A, const double *B, const double *C,
|
||||
}
|
||||
|
||||
// This warp's pointer to the C matrix data to copy memory from to shared memory.
|
||||
const size_t gmem_idx = (block_tile_i + warpId) * M * GLOBAL_MEM_STRIDE + block_tile_j * N;
|
||||
const size_t gmem_idx = (block_tile_i + warpId) * M * GLOBAL_MEM_STRIDE + block_tile_j * N;
|
||||
const double *src_gmem_warp_stream_ptr = &C[gmem_idx];
|
||||
|
||||
// Stream multiple C tiles to shared memory.
|
||||
@@ -271,7 +275,7 @@ __global__ void compute_dgemm(const double *A, const double *B, const double *C,
|
||||
|
||||
// Scale the C matrix.
|
||||
#pragma unroll
|
||||
for (int i = 0; i < WARP_COL_TILES; i++) {
|
||||
for (int i = 0; i < WARP_COL_TILES; i++) {
|
||||
#pragma unroll
|
||||
for (int j = 0; j < WARP_ROW_TILES; j++) {
|
||||
#pragma unroll
|
||||
@@ -283,16 +287,19 @@ __global__ void compute_dgemm(const double *A, const double *B, const double *C,
|
||||
|
||||
// Select what warp copies what matrix to shared memory.
|
||||
// Warps 0-3 copy the A matrix, warps 4-7 copy the B matrix.
|
||||
const double *warp_ptr = (warpId < (WARPS_PER_BLOCK/2)) ? (&A[block_tile_i * M * K_GLOBAL] + M * K_GLOBAL * (warpId % (WARPS_PER_BLOCK/2)) * 2) :
|
||||
(&B[block_tile_j * N * K_GLOBAL] + N * K_GLOBAL * (warpId % (WARPS_PER_BLOCK/2)) * 2);
|
||||
const double *warp_ptr =
|
||||
(warpId < (WARPS_PER_BLOCK / 2))
|
||||
? (&A[block_tile_i * M * K_GLOBAL] + M * K_GLOBAL * (warpId % (WARPS_PER_BLOCK / 2)) * 2)
|
||||
: (&B[block_tile_j * N * K_GLOBAL] + N * K_GLOBAL * (warpId % (WARPS_PER_BLOCK / 2)) * 2);
|
||||
|
||||
// Go through the global K dimension by a fixed step at a time.
|
||||
#pragma unroll
|
||||
for (int tile_k = 0; tile_k < K_TILES; tile_k += CHUNK_K) {
|
||||
// Copy slices of the A and B matrices to shared memory.
|
||||
// The first half of the warps in the CTA copy the A matrix, the rest copy the B matrix.
|
||||
size_t shmem_idx = warpId < (WARPS_PER_BLOCK/2) ? (M * (warpId % (WARPS_PER_BLOCK/2)) * 2) :
|
||||
(N * (warpId % (WARPS_PER_BLOCK/2)) * 2 + shmem_idx_b_off);
|
||||
size_t shmem_idx = warpId < (WARPS_PER_BLOCK / 2)
|
||||
? (M * (warpId % (WARPS_PER_BLOCK / 2)) * 2)
|
||||
: (N * (warpId % (WARPS_PER_BLOCK / 2)) * 2 + shmem_idx_b_off);
|
||||
|
||||
// First half of the warp copies the first row / column of the matrix,
|
||||
// the second half of the warp copies the next.
|
||||
@@ -302,9 +309,10 @@ __global__ void compute_dgemm(const double *A, const double *B, const double *C,
|
||||
shmem_idx += laneId / CHUNK_COPY_LINE_LANES;
|
||||
|
||||
#pragma unroll
|
||||
for(int i = 0; i < ((WARP_SIZE/2) / CHUNK_COPY_LINES_PER_WARP); i++) {
|
||||
// Copy 16 bytes at once in each lane.
|
||||
*((int4*)&shmem[shmem_idx][0] + (laneId % CHUNK_COPY_LINE_LANES)) = *((int4*)lane_ptr + (laneId % CHUNK_COPY_LINE_LANES));
|
||||
for (int i = 0; i < ((WARP_SIZE / 2) / CHUNK_COPY_LINES_PER_WARP); i++) {
|
||||
// Copy 16 bytes at once in each lane.
|
||||
*((int4 *)&shmem[shmem_idx][0] + (laneId % CHUNK_COPY_LINE_LANES)) =
|
||||
*((int4 *)lane_ptr + (laneId % CHUNK_COPY_LINE_LANES));
|
||||
|
||||
// Advance the global memory pointer and the shared memory index.
|
||||
lane_ptr = lane_ptr + K_GLOBAL * CHUNK_COPY_LINES_PER_WARP;
|
||||
@@ -321,8 +329,8 @@ __global__ void compute_dgemm(const double *A, const double *B, const double *C,
|
||||
|
||||
#pragma unroll
|
||||
for (int i = 0; i < WARP_COL_TILES; i++) {
|
||||
size_t shmem_idx_a = (warpId/2) * M * 2 + (i * M);
|
||||
const double *tile_ptr = &shmem[shmem_idx_a][k_step * K];
|
||||
size_t shmem_idx_a = (warpId / 2) * M * 2 + (i * M);
|
||||
const double *tile_ptr = &shmem[shmem_idx_a][k_step * K];
|
||||
|
||||
wmma::load_matrix_sync(a[i], tile_ptr, K * CHUNK_K + SKEW_DOUBLE);
|
||||
|
||||
@@ -331,11 +339,10 @@ __global__ void compute_dgemm(const double *A, const double *B, const double *C,
|
||||
if (i == 0) {
|
||||
// Load the B matrix fragment once, because it is going to be reused
|
||||
// against the other A matrix fragments.
|
||||
size_t shmem_idx_b = shmem_idx_b_off + (WARP_ROW_TILES * N) * (warpId%2) + (j * N);
|
||||
const double *tile_ptr = &shmem[shmem_idx_b][k_step * K];
|
||||
size_t shmem_idx_b = shmem_idx_b_off + (WARP_ROW_TILES * N) * (warpId % 2) + (j * N);
|
||||
const double *tile_ptr = &shmem[shmem_idx_b][k_step * K];
|
||||
|
||||
wmma::load_matrix_sync(b[j], tile_ptr, K * CHUNK_K + SKEW_DOUBLE);
|
||||
|
||||
}
|
||||
|
||||
wmma::mma_sync(c[i][j], a[i], b[j], c[i][j]);
|
||||
@@ -370,8 +377,8 @@ __global__ void compute_dgemm(const double *A, const double *B, const double *C,
|
||||
|
||||
#pragma unroll
|
||||
for (int i = 0; i < N; i++) {
|
||||
*((int4*)(dst_gmem_warp_stream_ptr + GLOBAL_MEM_STRIDE * i) + laneId) =
|
||||
*((int4*)(shmem_warp_stream_ptr + SHMEM_STRIDE * i) + laneId);
|
||||
*((int4 *)(dst_gmem_warp_stream_ptr + GLOBAL_MEM_STRIDE * i) + laneId) =
|
||||
*((int4 *)(shmem_warp_stream_ptr + SHMEM_STRIDE * i) + laneId);
|
||||
}
|
||||
|
||||
__syncthreads();
|
||||
@@ -379,7 +386,8 @@ __global__ void compute_dgemm(const double *A, const double *B, const double *C,
|
||||
#endif
|
||||
}
|
||||
|
||||
__global__ void compute_dgemm_async_copy(const double *A, const double *B, const double *C, double *D, double alpha, double beta)
|
||||
__global__ void
|
||||
compute_dgemm_async_copy(const double *A, const double *B, const double *C, double *D, double alpha, double beta)
|
||||
{
|
||||
#if __CUDA_ARCH__ >= 800
|
||||
extern __shared__ double shmem[][CHUNK_K * K + SKEW_DOUBLE];
|
||||
@@ -392,7 +400,8 @@ __global__ void compute_dgemm_async_copy(const double *A, const double *B, const
|
||||
constexpr size_t shmem_idx_b_off = BLOCK_COL_TILES * M;
|
||||
|
||||
// This pointer is used to access the C and D matrix tiles this warp computes.
|
||||
double *shmem_warp_tile_ptr = &shmem[0][0] + (warpId/BLOCK_ROW_WARPS) * SHMEM_STRIDE * N * BLOCK_ROW_WARPS + (warpId % BLOCK_ROW_WARPS) * SHMEM_OFFSET;
|
||||
double *shmem_warp_tile_ptr = &shmem[0][0] + (warpId / BLOCK_ROW_WARPS) * SHMEM_STRIDE * N * BLOCK_ROW_WARPS
|
||||
+ (warpId % BLOCK_ROW_WARPS) * SHMEM_OFFSET;
|
||||
|
||||
// This pointer is used to stream the C and D matrices block-wide tile to and from shared memory.
|
||||
double *shmem_warp_stream_ptr = &shmem[0][0] + warpId * SHMEM_STRIDE * N;
|
||||
@@ -404,13 +413,13 @@ __global__ void compute_dgemm_async_copy(const double *A, const double *B, const
|
||||
|
||||
cuda::pipeline<cuda::thread_scope_thread> pipe = cuda::make_pipeline();
|
||||
|
||||
const auto shape2 = cuda::aligned_size_t<alignof(double2)>(sizeof(double2));
|
||||
const auto shape2 = cuda::aligned_size_t<alignof(double2)>(sizeof(double2));
|
||||
constexpr int loadStride = 1; // load 2 double, left-shift by 1.
|
||||
|
||||
// Each CTA slides along the 64 x 64 tiles from the top left corner of the matrix to the
|
||||
// right and down, and selects the next tile to compute. Once there's no such tile,
|
||||
// all warps in this CTA exit.
|
||||
for(unsigned int block_pos = blockIdx.x;; block_pos += gridDim.x) {
|
||||
for (unsigned int block_pos = blockIdx.x;; block_pos += gridDim.x) {
|
||||
const unsigned int block_tile_i = ((block_pos * BLOCK_ROW_TILES) / N_TILES) * (BLOCK_COL_TILES);
|
||||
const unsigned int block_tile_j = (block_pos * BLOCK_COL_TILES) % N_TILES;
|
||||
|
||||
@@ -420,7 +429,7 @@ __global__ void compute_dgemm_async_copy(const double *A, const double *B, const
|
||||
}
|
||||
|
||||
// This warp's pointer to the C matrix data to copy memory from to shared memory.
|
||||
const size_t gmem_idx = (block_tile_i + warpId) * M * GLOBAL_MEM_STRIDE + block_tile_j * N;
|
||||
const size_t gmem_idx = (block_tile_i + warpId) * M * GLOBAL_MEM_STRIDE + block_tile_j * N;
|
||||
const double *src_gmem_warp_stream_ptr = &C[gmem_idx];
|
||||
|
||||
// Stream multiple C tiles to shared memory.
|
||||
@@ -428,8 +437,9 @@ __global__ void compute_dgemm_async_copy(const double *A, const double *B, const
|
||||
for (int i = 0; i < N; i++) {
|
||||
pipe.producer_acquire();
|
||||
cuda::memcpy_async(&shmem_warp_stream_ptr[(SHMEM_STRIDE * i) + (laneId << loadStride)],
|
||||
&src_gmem_warp_stream_ptr[(GLOBAL_MEM_STRIDE * i) + (laneId << loadStride)],
|
||||
shape2, pipe);
|
||||
&src_gmem_warp_stream_ptr[(GLOBAL_MEM_STRIDE * i) + (laneId << loadStride)],
|
||||
shape2,
|
||||
pipe);
|
||||
|
||||
pipe.producer_commit();
|
||||
}
|
||||
@@ -463,12 +473,14 @@ __global__ void compute_dgemm_async_copy(const double *A, const double *B, const
|
||||
|
||||
// Select what warp copies what matrix to shared memory.
|
||||
// Warps 0-3 copy the A matrix, warps 4-7 copy the B matrix.
|
||||
const double *warp_ptr = (warpId < (WARPS_PER_BLOCK/2)) ? (&A[block_tile_i * M * K_GLOBAL] + M * K_GLOBAL * (warpId % (WARPS_PER_BLOCK/2)) * 2) :
|
||||
(&B[block_tile_j * N * K_GLOBAL] + N * K_GLOBAL * (warpId % (WARPS_PER_BLOCK/2)) * 2);
|
||||
const double *warp_ptr =
|
||||
(warpId < (WARPS_PER_BLOCK / 2))
|
||||
? (&A[block_tile_i * M * K_GLOBAL] + M * K_GLOBAL * (warpId % (WARPS_PER_BLOCK / 2)) * 2)
|
||||
: (&B[block_tile_j * N * K_GLOBAL] + N * K_GLOBAL * (warpId % (WARPS_PER_BLOCK / 2)) * 2);
|
||||
|
||||
const int stridePerLaneCopy = (laneId / CHUNK_COPY_LINE_LANES);
|
||||
constexpr int chunksPerLane = ((WARP_SIZE/2) / CHUNK_COPY_LINES_PER_WARP);
|
||||
const int laneLoadElem = (laneId % CHUNK_COPY_LINE_LANES) << loadStride;
|
||||
const int stridePerLaneCopy = (laneId / CHUNK_COPY_LINE_LANES);
|
||||
constexpr int chunksPerLane = ((WARP_SIZE / 2) / CHUNK_COPY_LINES_PER_WARP);
|
||||
const int laneLoadElem = (laneId % CHUNK_COPY_LINE_LANES) << loadStride;
|
||||
|
||||
// Go through the global K dimension by a fixed step at a time.
|
||||
#pragma unroll
|
||||
@@ -476,7 +488,8 @@ __global__ void compute_dgemm_async_copy(const double *A, const double *B, const
|
||||
// Copy slices of the A and B matrices to shared memory.
|
||||
// The first half of the warps in the CTA copy the A matrix, the rest copy the B matrix.
|
||||
// As for DMMA M == N we use M for warp 4-7 + shmem_idx_b_off.
|
||||
size_t shmem_idx = (M * (warpId % (WARPS_PER_BLOCK/2)) * 2) + (shmem_idx_b_off * (warpId/(WARPS_PER_BLOCK/2)));
|
||||
size_t shmem_idx =
|
||||
(M * (warpId % (WARPS_PER_BLOCK / 2)) * 2) + (shmem_idx_b_off * (warpId / (WARPS_PER_BLOCK / 2)));
|
||||
|
||||
// First half of the warp copies the first row / column of the matrix,
|
||||
// the second half of the warp copies the next.
|
||||
@@ -485,8 +498,8 @@ __global__ void compute_dgemm_async_copy(const double *A, const double *B, const
|
||||
// Shift the second half of the warp to the next row / column in the shared memory.
|
||||
shmem_idx += stridePerLaneCopy;
|
||||
#pragma unroll
|
||||
for(int i = 0; i < chunksPerLane; i++) {
|
||||
// Copy 16 bytes at once in each lane.
|
||||
for (int i = 0; i < chunksPerLane; i++) {
|
||||
// Copy 16 bytes at once in each lane.
|
||||
pipe.producer_acquire();
|
||||
|
||||
cuda::memcpy_async(&shmem[shmem_idx][laneLoadElem], lane_ptr, shape2, pipe);
|
||||
@@ -508,8 +521,8 @@ __global__ void compute_dgemm_async_copy(const double *A, const double *B, const
|
||||
wmma::fragment<wmma::matrix_b, M, N, K, double, wmma::col_major> b[WARP_ROW_TILES];
|
||||
#pragma unroll
|
||||
for (int i = 0; i < WARP_COL_TILES; i++) {
|
||||
size_t shmem_idx_a = (warpId/2) * M * 2 + (i * M);
|
||||
const double *tile_ptr = &shmem[shmem_idx_a][k_step * K];
|
||||
size_t shmem_idx_a = (warpId / 2) * M * 2 + (i * M);
|
||||
const double *tile_ptr = &shmem[shmem_idx_a][k_step * K];
|
||||
|
||||
wmma::load_matrix_sync(a[i], tile_ptr, K * CHUNK_K + SKEW_DOUBLE);
|
||||
#pragma unroll
|
||||
@@ -517,8 +530,8 @@ __global__ void compute_dgemm_async_copy(const double *A, const double *B, const
|
||||
if (i == 0) {
|
||||
// Load the B matrix fragment once, because it is going to be reused
|
||||
// against the other A matrix fragments.
|
||||
size_t shmem_idx_b = shmem_idx_b_off + (WARP_ROW_TILES * N) * (warpId%2) + (j * N);
|
||||
const double *tile_ptr = &shmem[shmem_idx_b][k_step * K];
|
||||
size_t shmem_idx_b = shmem_idx_b_off + (WARP_ROW_TILES * N) * (warpId % 2) + (j * N);
|
||||
const double *tile_ptr = &shmem[shmem_idx_b][k_step * K];
|
||||
|
||||
wmma::load_matrix_sync(b[j], tile_ptr, K * CHUNK_K + SKEW_DOUBLE);
|
||||
}
|
||||
@@ -554,8 +567,8 @@ __global__ void compute_dgemm_async_copy(const double *A, const double *B, const
|
||||
|
||||
#pragma unroll
|
||||
for (int i = 0; i < N; i++) {
|
||||
*((int4*)(dst_gmem_warp_stream_ptr + GLOBAL_MEM_STRIDE * i) + laneId) =
|
||||
*((int4*)(shmem_warp_stream_ptr + SHMEM_STRIDE * i) + laneId);
|
||||
*((int4 *)(dst_gmem_warp_stream_ptr + GLOBAL_MEM_STRIDE * i) + laneId) =
|
||||
*((int4 *)(shmem_warp_stream_ptr + SHMEM_STRIDE * i) + laneId);
|
||||
}
|
||||
|
||||
__syncthreads();
|
||||
@@ -563,15 +576,16 @@ __global__ void compute_dgemm_async_copy(const double *A, const double *B, const
|
||||
#endif
|
||||
}
|
||||
|
||||
__global__ void compute_dgemm_cg_async_copy(const double *A, const double *B, const double *C, double *D, double alpha, double beta)
|
||||
__global__ void
|
||||
compute_dgemm_cg_async_copy(const double *A, const double *B, const double *C, double *D, double alpha, double beta)
|
||||
{
|
||||
#if __CUDA_ARCH__ >= 800
|
||||
extern __shared__ double shmem[][CHUNK_K * K + SKEW_DOUBLE];
|
||||
auto cta = cg::this_thread_block();
|
||||
auto tile32 = cg::tiled_partition<32>(cta);
|
||||
auto cta = cg::this_thread_block();
|
||||
auto tile32 = cg::tiled_partition<32>(cta);
|
||||
|
||||
constexpr int tileChunkCopySize = WARP_SIZE / CHUNK_COPY_LINES_PER_WARP;
|
||||
auto tileChunkCopy = cg::tiled_partition<tileChunkCopySize>(cta);
|
||||
auto tileChunkCopy = cg::tiled_partition<tileChunkCopySize>(cta);
|
||||
|
||||
// Warp and lane identification.
|
||||
const unsigned int warpId = threadIdx.x / WARP_SIZE;
|
||||
@@ -581,10 +595,11 @@ __global__ void compute_dgemm_async_copy(const double *A, const double *B, const
|
||||
constexpr size_t shmem_idx_b_off = BLOCK_COL_TILES * M;
|
||||
|
||||
// This pointer is used to access the C and D matrix tiles this warp computes.
|
||||
double *shmem_warp_tile_ptr = (double*)&shmem[0][0] + (warpId/2) * SHMEM_STRIDE * N * 2 + (warpId%2) * SHMEM_OFFSET;
|
||||
double *shmem_warp_tile_ptr =
|
||||
(double *)&shmem[0][0] + (warpId / 2) * SHMEM_STRIDE * N * 2 + (warpId % 2) * SHMEM_OFFSET;
|
||||
|
||||
// This pointer is used to stream the C and D matrices block-wide tile to and from shared memory.
|
||||
double *shmem_warp_stream_ptr = (double*)&shmem[0][0] + warpId * SHMEM_STRIDE * N;
|
||||
double *shmem_warp_stream_ptr = (double *)&shmem[0][0] + warpId * SHMEM_STRIDE * N;
|
||||
|
||||
// Adjust the beta scaler, as it'll be multiplied by alpha at the end of
|
||||
// each tile computation. Technically this is not generally correct (may result
|
||||
@@ -594,7 +609,7 @@ __global__ void compute_dgemm_async_copy(const double *A, const double *B, const
|
||||
// Each CTA slides along the 64 x 64 tiles from the top left corner of the matrix to the
|
||||
// right and down, and selects the next tile to compute. Once there's no such tile,
|
||||
// all warps in this CTA exit.
|
||||
for(unsigned int block_pos = blockIdx.x;; block_pos += gridDim.x) {
|
||||
for (unsigned int block_pos = blockIdx.x;; block_pos += gridDim.x) {
|
||||
const unsigned int block_tile_i = ((block_pos * BLOCK_ROW_TILES) / N_TILES) * (BLOCK_COL_TILES);
|
||||
const unsigned int block_tile_j = (block_pos * BLOCK_COL_TILES) % N_TILES;
|
||||
|
||||
@@ -604,7 +619,7 @@ __global__ void compute_dgemm_async_copy(const double *A, const double *B, const
|
||||
}
|
||||
|
||||
// This warp's pointer to the C matrix data to copy memory from to shared memory.
|
||||
const size_t gmem_idx = (block_tile_i + warpId) * M * GLOBAL_MEM_STRIDE + block_tile_j * N;
|
||||
const size_t gmem_idx = (block_tile_i + warpId) * M * GLOBAL_MEM_STRIDE + block_tile_j * N;
|
||||
const double *src_gmem_warp_stream_ptr = &C[gmem_idx];
|
||||
|
||||
// Stream multiple C tiles to shared memory.
|
||||
@@ -612,7 +627,8 @@ __global__ void compute_dgemm_async_copy(const double *A, const double *B, const
|
||||
for (int i = 0; i < N; i++) {
|
||||
auto dst_ptr = &shmem_warp_stream_ptr[(SHMEM_STRIDE * i)];
|
||||
auto src_ptr = &src_gmem_warp_stream_ptr[(GLOBAL_MEM_STRIDE * i)];
|
||||
cg::memcpy_async(tile32, dst_ptr, src_ptr, cuda::aligned_size_t<alignof(double2)>{tile32.size() * sizeof(double2)});
|
||||
cg::memcpy_async(
|
||||
tile32, dst_ptr, src_ptr, cuda::aligned_size_t<alignof(double2)>{tile32.size() * sizeof(double2)});
|
||||
}
|
||||
|
||||
cg::wait(cta);
|
||||
@@ -647,18 +663,20 @@ __global__ void compute_dgemm_async_copy(const double *A, const double *B, const
|
||||
cg::wait(cta);
|
||||
// Select what warp copies what matrix to shared memory.
|
||||
// Warps 0-3 copy the A matrix, warps 4-7 copy the B matrix.
|
||||
const double *warp_ptr = (warpId < 4) ? (&A[block_tile_i * M * K_GLOBAL] + M * K_GLOBAL * (warpId % (WARPS_PER_BLOCK/2)) * 2) :
|
||||
(&B[block_tile_j * N * K_GLOBAL] + N * K_GLOBAL * (warpId % (WARPS_PER_BLOCK/2)) * 2);
|
||||
const double *warp_ptr =
|
||||
(warpId < 4) ? (&A[block_tile_i * M * K_GLOBAL] + M * K_GLOBAL * (warpId % (WARPS_PER_BLOCK / 2)) * 2)
|
||||
: (&B[block_tile_j * N * K_GLOBAL] + N * K_GLOBAL * (warpId % (WARPS_PER_BLOCK / 2)) * 2);
|
||||
|
||||
const int stridePerLaneCopy = (laneId / CHUNK_COPY_LINE_LANES);
|
||||
constexpr int chunksPerLane = ((WARP_SIZE/2) / CHUNK_COPY_LINES_PER_WARP);
|
||||
const int stridePerLaneCopy = (laneId / CHUNK_COPY_LINE_LANES);
|
||||
constexpr int chunksPerLane = ((WARP_SIZE / 2) / CHUNK_COPY_LINES_PER_WARP);
|
||||
// Go through the global K dimension by a fixed step at a time.
|
||||
#pragma unroll
|
||||
for (int tile_k = 0; tile_k < K_TILES; tile_k += CHUNK_K) {
|
||||
// Copy slices of the A and B matrices to shared memory.
|
||||
// The first half of the warps in the CTA copy the A matrix, the rest copy the B matrix.
|
||||
// As for DMMA M == N we use M for warp 4-7 + shmem_idx_b_off.
|
||||
size_t shmem_idx = (M * (warpId % (WARPS_PER_BLOCK/2)) * 2) + (shmem_idx_b_off * (warpId/(WARPS_PER_BLOCK/2)));
|
||||
size_t shmem_idx =
|
||||
(M * (warpId % (WARPS_PER_BLOCK / 2)) * 2) + (shmem_idx_b_off * (warpId / (WARPS_PER_BLOCK / 2)));
|
||||
|
||||
// First half of the warp copies the first row / column of the matrix,
|
||||
// the second half of the warp copies the next.
|
||||
@@ -668,13 +686,15 @@ __global__ void compute_dgemm_async_copy(const double *A, const double *B, const
|
||||
shmem_idx += stridePerLaneCopy;
|
||||
|
||||
#pragma unroll
|
||||
for(int i = 0; i < chunksPerLane; i++) {
|
||||
for (int i = 0; i < chunksPerLane; i++) {
|
||||
// Copy 16 bytes at once in each lane.
|
||||
auto dst_ptr = &shmem[shmem_idx][0];
|
||||
auto src_ptr = lane_ptr;
|
||||
|
||||
cg::memcpy_async(tileChunkCopy, dst_ptr, src_ptr,
|
||||
cuda::aligned_size_t<alignof(double2)>{tileChunkCopySize * sizeof(double2)});
|
||||
cg::memcpy_async(tileChunkCopy,
|
||||
dst_ptr,
|
||||
src_ptr,
|
||||
cuda::aligned_size_t<alignof(double2)>{tileChunkCopySize * sizeof(double2)});
|
||||
|
||||
// Advance the global memory pointer and the shared memory index.
|
||||
lane_ptr = lane_ptr + K_GLOBAL * CHUNK_COPY_LINES_PER_WARP;
|
||||
@@ -690,8 +710,8 @@ __global__ void compute_dgemm_async_copy(const double *A, const double *B, const
|
||||
|
||||
#pragma unroll
|
||||
for (int i = 0; i < WARP_COL_TILES; i++) {
|
||||
size_t shmem_idx_a = (warpId/2) * M * 2 + (i * M);
|
||||
const double *tile_ptr = &shmem[shmem_idx_a][k_step * K];
|
||||
size_t shmem_idx_a = (warpId / 2) * M * 2 + (i * M);
|
||||
const double *tile_ptr = &shmem[shmem_idx_a][k_step * K];
|
||||
|
||||
wmma::load_matrix_sync(a[i], tile_ptr, K * CHUNK_K + SKEW_DOUBLE);
|
||||
|
||||
@@ -700,11 +720,10 @@ __global__ void compute_dgemm_async_copy(const double *A, const double *B, const
|
||||
if (i == 0) {
|
||||
// Load the B matrix fragment once, because it is going to be reused
|
||||
// against the other A matrix fragments.
|
||||
size_t shmem_idx_b = shmem_idx_b_off + (WARP_ROW_TILES * N) * (warpId%2) + (j * N);
|
||||
const double *tile_ptr = &shmem[shmem_idx_b][k_step * K];
|
||||
size_t shmem_idx_b = shmem_idx_b_off + (WARP_ROW_TILES * N) * (warpId % 2) + (j * N);
|
||||
const double *tile_ptr = &shmem[shmem_idx_b][k_step * K];
|
||||
|
||||
wmma::load_matrix_sync(b[j], tile_ptr, K * CHUNK_K + SKEW_DOUBLE);
|
||||
|
||||
}
|
||||
|
||||
wmma::mma_sync(c[i][j], a[i], b[j], c[i][j]);
|
||||
@@ -738,8 +757,8 @@ __global__ void compute_dgemm_async_copy(const double *A, const double *B, const
|
||||
|
||||
#pragma unroll
|
||||
for (int i = 0; i < N; i++) {
|
||||
*((int4*)(dst_gmem_warp_stream_ptr + GLOBAL_MEM_STRIDE * i) + laneId) =
|
||||
*((int4*)(shmem_warp_stream_ptr + SHMEM_STRIDE * i) + laneId);
|
||||
*((int4 *)(dst_gmem_warp_stream_ptr + GLOBAL_MEM_STRIDE * i) + laneId) =
|
||||
*((int4 *)(shmem_warp_stream_ptr + SHMEM_STRIDE * i) + laneId);
|
||||
}
|
||||
cg::sync(cta);
|
||||
}
|
||||
@@ -748,12 +767,13 @@ __global__ void compute_dgemm_async_copy(const double *A, const double *B, const
|
||||
|
||||
// Performs an MxNxK DGEMM (C=alpha*A*B + beta*C) assuming:
|
||||
// 1) Matrices are packed in memory.
|
||||
// 2) M, N and K are multiples of 8, 8 and 4 respectively.
|
||||
// 2) M, N and K are multiples of 8, 8 and 4 respectively.
|
||||
// 3) A is row major, B is column major matrix.
|
||||
// Note: This is a less performant version of the compute_dgemm kernel. It is designed for
|
||||
// demonstration purposes only to show the CUDA WMMA API use without relying on
|
||||
// availability of the shared memory.
|
||||
__global__ void simple_wmma_gemm(double *a, double *b, double *c, double *d, int m_ld, int n_ld, int k_ld, double alpha, double beta)
|
||||
__global__ void
|
||||
simple_wmma_gemm(double *a, double *b, double *c, double *d, int m_ld, int n_ld, int k_ld, double alpha, double beta)
|
||||
{
|
||||
#if __CUDA_ARCH__ >= 800
|
||||
// Leading dimensions. Packed with no transpositions.
|
||||
@@ -768,8 +788,8 @@ __global__ void simple_wmma_gemm(double *a, double *b, double *c, double *d, int
|
||||
// Declare the fragments
|
||||
wmma::fragment<wmma::matrix_a, M, N, K, double, wmma::row_major> a_frag;
|
||||
wmma::fragment<wmma::matrix_b, M, N, K, double, wmma::col_major> b_frag;
|
||||
wmma::fragment<wmma::accumulator, M, N, K, double> acc_frag;
|
||||
wmma::fragment<wmma::accumulator, M, N, K, double> c_frag;
|
||||
wmma::fragment<wmma::accumulator, M, N, K, double> acc_frag;
|
||||
wmma::fragment<wmma::accumulator, M, N, K, double> c_frag;
|
||||
|
||||
wmma::fill_fragment(acc_frag, 0.0f);
|
||||
|
||||
@@ -799,7 +819,7 @@ __global__ void simple_wmma_gemm(double *a, double *b, double *c, double *d, int
|
||||
if (cRow < m_ld && cCol < n_ld) {
|
||||
wmma::load_matrix_sync(c_frag, c + cCol + cRow * ldc, ldc, wmma::mem_row_major);
|
||||
|
||||
for(int i=0; i < c_frag.num_elements; i++) {
|
||||
for (int i = 0; i < c_frag.num_elements; i++) {
|
||||
c_frag.x[i] = alpha * acc_frag.x[i] + beta * c_frag.x[i];
|
||||
}
|
||||
|
||||
@@ -809,11 +829,17 @@ __global__ void simple_wmma_gemm(double *a, double *b, double *c, double *d, int
|
||||
#endif
|
||||
}
|
||||
|
||||
__host__ void matMultiplyOnHost(double *A, double *B, double *C,
|
||||
float alpha, float beta,
|
||||
int numARows, int numAColumns,
|
||||
int numBRows, int numBColumns,
|
||||
int numCRows, int numCColumns)
|
||||
__host__ void matMultiplyOnHost(double *A,
|
||||
double *B,
|
||||
double *C,
|
||||
float alpha,
|
||||
float beta,
|
||||
int numARows,
|
||||
int numAColumns,
|
||||
int numBRows,
|
||||
int numBColumns,
|
||||
int numCRows,
|
||||
int numCColumns)
|
||||
{
|
||||
for (int i = 0; i < numCRows; i++) {
|
||||
for (int j = 0; j < numCColumns; j++) {
|
||||
@@ -824,7 +850,7 @@ __host__ void matMultiplyOnHost(double *A, double *B, double *C,
|
||||
temp += A[i * numAColumns + k] * B[j * numBRows + k];
|
||||
}
|
||||
|
||||
C[i*numCColumns + j] = temp * alpha + beta * C[i * numCColumns + j];
|
||||
C[i * numCColumns + j] = temp * alpha + beta * C[i * numCColumns + j];
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -852,16 +878,16 @@ int main(int argc, char **argv)
|
||||
double *B_h = NULL;
|
||||
double *C_h = NULL;
|
||||
#if CPU_DEBUG
|
||||
double *result_hD = NULL;
|
||||
double *result_hD = NULL;
|
||||
double *result_host = NULL;
|
||||
#endif
|
||||
|
||||
A_h = (double*) malloc(sizeof(double) * M_GLOBAL * K_GLOBAL);
|
||||
B_h = (double*) malloc(sizeof(double) * K_GLOBAL * N_GLOBAL);
|
||||
C_h = (double*) malloc(sizeof(double) * M_GLOBAL * N_GLOBAL);
|
||||
A_h = (double *)malloc(sizeof(double) * M_GLOBAL * K_GLOBAL);
|
||||
B_h = (double *)malloc(sizeof(double) * K_GLOBAL * N_GLOBAL);
|
||||
C_h = (double *)malloc(sizeof(double) * M_GLOBAL * N_GLOBAL);
|
||||
#if CPU_DEBUG
|
||||
result_hD = (double*) malloc(sizeof(double) * M_GLOBAL * N_GLOBAL);
|
||||
result_host = (double*) malloc(sizeof(double) * M_GLOBAL * N_GLOBAL);
|
||||
result_hD = (double *)malloc(sizeof(double) * M_GLOBAL * N_GLOBAL);
|
||||
result_host = (double *)malloc(sizeof(double) * M_GLOBAL * N_GLOBAL);
|
||||
#endif
|
||||
|
||||
double *A = NULL;
|
||||
@@ -869,10 +895,10 @@ int main(int argc, char **argv)
|
||||
double *C = NULL;
|
||||
double *D = NULL;
|
||||
|
||||
checkCudaErrors(cudaMalloc((void**)&A, sizeof(double) * M_GLOBAL * K_GLOBAL));
|
||||
checkCudaErrors(cudaMalloc((void**)&B, sizeof(double) * N_GLOBAL * K_GLOBAL));
|
||||
checkCudaErrors(cudaMalloc((void**)&C, sizeof(double) * M_GLOBAL * N_GLOBAL));
|
||||
checkCudaErrors(cudaMalloc((void**)&D, sizeof(double) * M_GLOBAL * N_GLOBAL));
|
||||
checkCudaErrors(cudaMalloc((void **)&A, sizeof(double) * M_GLOBAL * K_GLOBAL));
|
||||
checkCudaErrors(cudaMalloc((void **)&B, sizeof(double) * N_GLOBAL * K_GLOBAL));
|
||||
checkCudaErrors(cudaMalloc((void **)&C, sizeof(double) * M_GLOBAL * N_GLOBAL));
|
||||
checkCudaErrors(cudaMalloc((void **)&D, sizeof(double) * M_GLOBAL * N_GLOBAL));
|
||||
|
||||
assert(((unsigned long long)A) % 128 == 0);
|
||||
assert(((unsigned long long)B) % 128 == 0);
|
||||
@@ -900,7 +926,7 @@ int main(int argc, char **argv)
|
||||
printf("Required shared memory size: %lu Kb\n", SHMEM_SZ / 1024UL);
|
||||
|
||||
const double alpha = 1.1f;
|
||||
const double beta = 1.2f;
|
||||
const double beta = 1.2f;
|
||||
|
||||
cudaEvent_t start, stop;
|
||||
|
||||
@@ -913,45 +939,47 @@ int main(int argc, char **argv)
|
||||
// kernel to run - default (dmma_shmem_gemm_async_copy == 0)
|
||||
if (checkCmdLineFlag(argc, (const char **)argv, "kernel")) {
|
||||
int kernel_number = getCmdLineArgumentInt(argc, (const char **)argv, "kernel");
|
||||
if (kernel_number < 4)
|
||||
{
|
||||
if (kernel_number < 4) {
|
||||
selected_kernel = (kernels)kernel_number;
|
||||
}
|
||||
else
|
||||
{
|
||||
else {
|
||||
printf("Error: kernel number should be between 0 to 3, you have entered %d\n", kernel_number);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
}
|
||||
|
||||
// If enough shared memory available on the GPU use high performant kernel
|
||||
if ((deviceProp.sharedMemPerMultiprocessor >= SHMEM_SZ) && (selected_kernel != simple_dmma_gemm))
|
||||
{
|
||||
if ((deviceProp.sharedMemPerMultiprocessor >= SHMEM_SZ) && (selected_kernel != simple_dmma_gemm)) {
|
||||
printf("Computing using high performance kernel = %d - %s\n", selected_kernel, kernelNames[selected_kernel]);
|
||||
|
||||
switch (selected_kernel)
|
||||
{
|
||||
case dmma_shmem_gemm_async_copy :
|
||||
default:
|
||||
checkCudaErrors(cudaFuncSetAttribute(compute_dgemm_async_copy, cudaFuncAttributeMaxDynamicSharedMemorySize, SHMEM_SZ));
|
||||
checkKernelErrors((compute_dgemm_async_copy<<<deviceProp.multiProcessorCount*3, THREADS_PER_BLOCK, SHMEM_SZ>>>(A, B, C, D, alpha, beta)));
|
||||
break;
|
||||
case dmma_shmem_gemm_cg_async_copy :
|
||||
checkCudaErrors(cudaFuncSetAttribute(compute_dgemm_cg_async_copy, cudaFuncAttributeMaxDynamicSharedMemorySize, SHMEM_SZ));
|
||||
checkKernelErrors((compute_dgemm_cg_async_copy<<<deviceProp.multiProcessorCount*3, THREADS_PER_BLOCK, SHMEM_SZ>>>(A, B, C, D, alpha, beta)));
|
||||
break;
|
||||
case dmma_shmem_gemm :
|
||||
checkCudaErrors(cudaFuncSetAttribute(compute_dgemm, cudaFuncAttributeMaxDynamicSharedMemorySize, SHMEM_SZ));
|
||||
checkKernelErrors((compute_dgemm<<<deviceProp.multiProcessorCount*2, THREADS_PER_BLOCK, SHMEM_SZ>>>(A, B, C, D, alpha, beta)));
|
||||
break;
|
||||
switch (selected_kernel) {
|
||||
case dmma_shmem_gemm_async_copy:
|
||||
default:
|
||||
checkCudaErrors(
|
||||
cudaFuncSetAttribute(compute_dgemm_async_copy, cudaFuncAttributeMaxDynamicSharedMemorySize, SHMEM_SZ));
|
||||
checkKernelErrors(
|
||||
(compute_dgemm_async_copy<<<deviceProp.multiProcessorCount * 3, THREADS_PER_BLOCK, SHMEM_SZ>>>(
|
||||
A, B, C, D, alpha, beta)));
|
||||
break;
|
||||
case dmma_shmem_gemm_cg_async_copy:
|
||||
checkCudaErrors(cudaFuncSetAttribute(
|
||||
compute_dgemm_cg_async_copy, cudaFuncAttributeMaxDynamicSharedMemorySize, SHMEM_SZ));
|
||||
checkKernelErrors(
|
||||
(compute_dgemm_cg_async_copy<<<deviceProp.multiProcessorCount * 3, THREADS_PER_BLOCK, SHMEM_SZ>>>(
|
||||
A, B, C, D, alpha, beta)));
|
||||
break;
|
||||
case dmma_shmem_gemm:
|
||||
checkCudaErrors(cudaFuncSetAttribute(compute_dgemm, cudaFuncAttributeMaxDynamicSharedMemorySize, SHMEM_SZ));
|
||||
checkKernelErrors((compute_dgemm<<<deviceProp.multiProcessorCount * 2, THREADS_PER_BLOCK, SHMEM_SZ>>>(
|
||||
A, B, C, D, alpha, beta)));
|
||||
break;
|
||||
}
|
||||
|
||||
#if CPU_DEBUG
|
||||
checkCudaErrors(cudaMemcpy(result_hD, D, sizeof(double)*M_GLOBAL*N_GLOBAL, cudaMemcpyDeviceToHost));
|
||||
checkCudaErrors(cudaMemcpy(result_hD, D, sizeof(double) * M_GLOBAL * N_GLOBAL, cudaMemcpyDeviceToHost));
|
||||
#endif
|
||||
}
|
||||
else
|
||||
{
|
||||
else {
|
||||
dim3 gridDim;
|
||||
dim3 blockDim;
|
||||
|
||||
@@ -978,25 +1006,19 @@ int main(int argc, char **argv)
|
||||
|
||||
memcpy(result_host, C_h, sizeof(double) * M_GLOBAL * N_GLOBAL);
|
||||
|
||||
matMultiplyOnHost(A_h, B_h, result_host,
|
||||
alpha, beta,
|
||||
M_GLOBAL, K_GLOBAL,
|
||||
K_GLOBAL, N_GLOBAL,
|
||||
M_GLOBAL, N_GLOBAL);
|
||||
matMultiplyOnHost(A_h, B_h, result_host, alpha, beta, M_GLOBAL, K_GLOBAL, K_GLOBAL, N_GLOBAL, M_GLOBAL, N_GLOBAL);
|
||||
|
||||
size_t number_of_matches = 0;
|
||||
for (int i = 0; i < N_GLOBAL*M_GLOBAL; i++) {
|
||||
if (fabs(result_hD[i] - result_host[i]) > 0.1f)
|
||||
{
|
||||
for (int i = 0; i < N_GLOBAL * M_GLOBAL; i++) {
|
||||
if (fabs(result_hD[i] - result_host[i]) > 0.1f) {
|
||||
printf("mismatch i=%d result_hD=%f result_host=%f\n", i, result_hD[i], result_host[i]);
|
||||
break;
|
||||
}
|
||||
else
|
||||
{
|
||||
else {
|
||||
number_of_matches++;
|
||||
}
|
||||
}
|
||||
printf("number_of_matches = %zu out of = %d \n", number_of_matches, N_GLOBAL*M_GLOBAL);
|
||||
printf("number_of_matches = %zu out of = %d \n", number_of_matches, N_GLOBAL * M_GLOBAL);
|
||||
free(result_hD);
|
||||
free(result_host);
|
||||
#endif
|
||||
@@ -1006,15 +1028,15 @@ int main(int argc, char **argv)
|
||||
checkCudaErrors(cudaEventElapsedTime(&milliseconds, start, stop));
|
||||
|
||||
printf("Time: %f ms\n", milliseconds);
|
||||
printf("FP64 TFLOPS: %.2f\n", (((double)M_GLOBAL * N_GLOBAL * K_GLOBAL * 2)/(milliseconds/1000.)) / 1e12);
|
||||
printf("FP64 TFLOPS: %.2f\n", (((double)M_GLOBAL * N_GLOBAL * K_GLOBAL * 2) / (milliseconds / 1000.)) / 1e12);
|
||||
|
||||
free(A_h);
|
||||
free(B_h);
|
||||
free(C_h);
|
||||
checkCudaErrors(cudaFree((void*)A));
|
||||
checkCudaErrors(cudaFree((void*)B));
|
||||
checkCudaErrors(cudaFree((void*)C));
|
||||
checkCudaErrors(cudaFree((void*)D));
|
||||
checkCudaErrors(cudaFree((void *)A));
|
||||
checkCudaErrors(cudaFree((void *)B));
|
||||
checkCudaErrors(cudaFree((void *)C));
|
||||
checkCudaErrors(cudaFree((void *)D));
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user