mirror of
https://github.com/NVIDIA/cuda-samples.git
synced 2026-11-18 01:48:16 +08:00
776 lines
35 KiB
Plaintext
776 lines
35 KiB
Plaintext
/* Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
|
|
*
|
|
* Redistribution and use in source and binary forms, with or without
|
|
* modification, are permitted provided that the following conditions
|
|
* are met:
|
|
* * Redistributions of source code must retain the above copyright
|
|
* notice, this list of conditions and the following disclaimer.
|
|
* * Redistributions in binary form must reproduce the above copyright
|
|
* notice, this list of conditions and the following disclaimer in the
|
|
* documentation and/or other materials provided with the distribution.
|
|
* * Neither the name of NVIDIA CORPORATION nor the names of its
|
|
* contributors may be used to endorse or promote products derived
|
|
* from this software without specific prior written permission.
|
|
*
|
|
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
|
|
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
|
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
|
|
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
|
|
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
|
|
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
|
|
* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
|
|
* OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
*/
|
|
|
|
/*
|
|
* Unicast point-to-point data movement with Compute Fabric Transport (CFT)
|
|
* logical endpoints.
|
|
*
|
|
* MPI example on a ring: rank r sends to (r+1)%N and receives from (r-1+N)%N.
|
|
* Default mode uses plain fabric puts; a separate one-thread kernel then flags the
|
|
* send neighbor over the same endpoint and waits on the receive neighbor's flag.
|
|
* Pass --counted for counted fabric puts and a counter which counts bytes arrival
|
|
* for receive completion.
|
|
*/
|
|
|
|
#include <cassert>
|
|
#include <climits>
|
|
#include <cstddef>
|
|
#include <cuda.h>
|
|
#include <cuda/atomic>
|
|
#include <cuda/ptx>
|
|
#include <cuda_awbarrier_helpers.h>
|
|
#include <cuda_runtime.h>
|
|
#include <memory>
|
|
#include <mpi.h>
|
|
#include <numeric>
|
|
#include <stdio.h>
|
|
#include <stdlib.h>
|
|
#include <string.h>
|
|
#include <vector>
|
|
|
|
#ifndef EXIT_WAIVED
|
|
#define EXIT_WAIVED 2
|
|
#endif
|
|
|
|
#define MPI_CHECK(cmd) \
|
|
do { \
|
|
int e = cmd; \
|
|
if (e != MPI_SUCCESS) { \
|
|
printf("Failed: MPI error %s:%d '%d'\n", __FILE__, __LINE__, e); \
|
|
exit(EXIT_FAILURE); \
|
|
} \
|
|
} while (0)
|
|
|
|
static void checkCuda(CUresult result, const char *func, const char *file, int line)
|
|
{
|
|
if (result != CUDA_SUCCESS) {
|
|
const char *name = nullptr;
|
|
cuGetErrorName(result, &name);
|
|
fprintf(stderr,
|
|
"CUDA driver error at %s:%d code=%d(%s) \"%s\"\n",
|
|
file,
|
|
line,
|
|
static_cast<int>(result),
|
|
name ? name : "<unknown>",
|
|
func);
|
|
exit(EXIT_FAILURE);
|
|
}
|
|
}
|
|
|
|
static void checkCuda(cudaError_t result, const char *func, const char *file, int line)
|
|
{
|
|
if (result != cudaSuccess) {
|
|
fprintf(stderr,
|
|
"CUDA runtime error at %s:%d code=%d(%s) \"%s\"\n",
|
|
file,
|
|
line,
|
|
static_cast<int>(result),
|
|
cudaGetErrorName(result),
|
|
func);
|
|
exit(EXIT_FAILURE);
|
|
}
|
|
}
|
|
|
|
#define checkCudaErrors(val) checkCuda((val), #val, __FILE__, __LINE__)
|
|
|
|
// Round value up to the next multiple of alignment (alignment need not be a power of two).
|
|
constexpr size_t alignUp(size_t value, size_t alignment)
|
|
{
|
|
return (value + alignment - 1) / alignment * alignment;
|
|
}
|
|
|
|
void waitForEndpointReady(CUlogicalEndpointId leId)
|
|
{
|
|
while (true) {
|
|
int ready = 0;
|
|
checkCudaErrors(cuLogicalEndpointQuery(leId, 1, &ready));
|
|
if (ready > 0)
|
|
break;
|
|
}
|
|
}
|
|
|
|
static constexpr uint32_t FABRIC_CHUNK_SIZE = 8192;
|
|
// A b128 fabric put moves 16 bytes at a time, and mbarrier complete_tx::16B
|
|
// counts transferred bytes in 16-byte units. So an expected-tx count is always
|
|
// the byte count divided by this granularity.
|
|
static constexpr uint32_t FABRIC_TX_GRANULARITY_BYTES = 16;
|
|
// Sync region placed after the payload on a 256-byte boundary, reused per mode:
|
|
// default : a 16-byte inter-GPU flag that the receive neighbor sets with a
|
|
// fabric put (b128 puts move 16 bytes at a time).
|
|
// --counted : an 8-byte delivered-byte counter.
|
|
static constexpr size_t INTER_GPU_FLAG_BYTES = 16;
|
|
static constexpr size_t SYNC_REGION_BYTES = 16; // max of counter (8B) and flag (16B)
|
|
// The counter targeted by counted fabric puts must be 256-byte aligned, so the
|
|
// whole sync region is placed on a 256-byte boundary.
|
|
static constexpr size_t SYNC_REGION_ALIGN_BYTES = 256;
|
|
static constexpr int WARP_SIZE = 32;
|
|
|
|
#ifndef NDEBUG
|
|
// Debug builds: cap spin-waits at 30 s so a stuck fabric op prints instead of hanging.
|
|
static constexpr uint64_t CFT_WAIT_TIMEOUT_NS = 30ULL * 1000 * 1000 * 1000;
|
|
|
|
inline __device__ uint64_t readGlobalTimerNs()
|
|
{
|
|
uint64_t t = 0;
|
|
asm volatile("mov.u64 %0, %%globaltimer;" : "=l"(t));
|
|
return t;
|
|
}
|
|
#endif
|
|
|
|
// Spin while pred() is true. Returns true when pred() becomes false; in debug
|
|
// builds returns false after CFT_WAIT_TIMEOUT_NS and prints `what` to identify
|
|
// the stall. Release builds spin forever. Caller must handle false (e.g. __trap()).
|
|
template <typename Predicate>
|
|
inline __device__ bool waitWithDebugTimeout(Predicate &&pred, const char *what)
|
|
{
|
|
#ifndef NDEBUG
|
|
uint64_t start = readGlobalTimerNs();
|
|
while (pred()) {
|
|
uint64_t now = readGlobalTimerNs();
|
|
if (now - start > CFT_WAIT_TIMEOUT_NS) {
|
|
printf("%s timeout after %llu ns (block=%u)\n", what, (unsigned long long)(now - start), blockIdx.x);
|
|
return false;
|
|
}
|
|
}
|
|
#else
|
|
(void)what;
|
|
while (pred()) {
|
|
}
|
|
#endif
|
|
return true;
|
|
}
|
|
|
|
// Decode and print the fabric errors captured in the mbarrier's report value.
|
|
// Decode it with cudaFabricOpStatusSourceMbarrierV1: first query the error count, then
|
|
// retrieve each decoded error. Call this only when the report predicate is set.
|
|
inline __device__ void reportFabricError(uint8_t *reportValue, uint64_t offset)
|
|
{
|
|
const unsigned long long off = offset;
|
|
unsigned int errCount = 0;
|
|
if (cudaFabricOpErrorStatusCount(reportValue, cudaFabricOpStatusSourceMbarrierV1, &errCount) != cudaSuccess) {
|
|
printf("cudaFabricOpErrorStatusCount failed at offset %llu\n", off);
|
|
return;
|
|
}
|
|
for (unsigned int i = 0; i < errCount; ++i) {
|
|
cudaFabricOpStatusInfo info = cudaFabricOpStatusInfoSuccess;
|
|
if (cudaFabricOpErrorStatusGet(reportValue, cudaFabricOpStatusSourceMbarrierV1, i, &info) == cudaSuccess) {
|
|
printf("fabric put error[%u/%u] at offset %llu: status=%d\n", i, errCount, off, (int)info);
|
|
}
|
|
else {
|
|
printf("cudaFabricOpErrorStatusGet failed for error[%u/%u] at offset %llu\n", i, errCount, off);
|
|
}
|
|
}
|
|
}
|
|
|
|
// Each block sends one FABRIC_CHUNK_SIZE slice to the peer's logical endpoint (putLeId).
|
|
// blockIdx.x selects the chunk: offset = blockIdx.x * FABRIC_CHUNK_SIZE. Launch
|
|
// gridDim.x = totalBytes / FABRIC_CHUNK_SIZE so every chunk is covered by one block.
|
|
//
|
|
// expectedBytes is how many bytes this rank expects to receive (tracked in our
|
|
// local counter by the peer's puts).
|
|
__global__ void cftTryPutCountedKernel(uint32_t putLeId,
|
|
uint64_t counterOffset,
|
|
int myRank,
|
|
uint64_t *counterPtr,
|
|
uint64_t expectedBytes)
|
|
{
|
|
constexpr int elems = FABRIC_CHUNK_SIZE / sizeof(int);
|
|
__shared__ alignas(16) int smemBuf[elems];
|
|
__shared__ __mbarrier_t putBar;
|
|
|
|
if (threadIdx.x == 0) {
|
|
cuda::ptx::mbarrier_init(cuda::ptx::layout_v1, &putBar, 1);
|
|
}
|
|
|
|
// Fill shared memory with a rank-tagged pattern before streaming it to the peer.
|
|
for (int i = threadIdx.x; i < elems; i += blockDim.x) {
|
|
smemBuf[i] = myRank * elems + i;
|
|
}
|
|
__syncthreads();
|
|
|
|
// Thread 0 issues one counted fabric put for this block's chunk.
|
|
if (threadIdx.x == 0) {
|
|
const uint64_t offset = blockIdx.x * FABRIC_CHUNK_SIZE;
|
|
// Make the staged shared-memory source visible to the async proxy before the put reads it.
|
|
cuda::ptx::fence_proxy_async(cuda::ptx::space_shared);
|
|
cuda::ptx::fabric_try_put_counted(cuda::ptx::space_shared,
|
|
cuda::ptx::sem_relaxed,
|
|
cuda::ptx::scope_sys,
|
|
putLeId,
|
|
offset,
|
|
counterOffset,
|
|
smemBuf,
|
|
FABRIC_CHUNK_SIZE,
|
|
&putBar);
|
|
cuda::ptx::fabric_submit();
|
|
cuda::ptx::mbarrier_arrive_expect_tx(cuda::ptx::sem_relaxed,
|
|
cuda::ptx::scope_cta,
|
|
cuda::ptx::space_shared,
|
|
&putBar,
|
|
(FABRIC_CHUNK_SIZE / FABRIC_TX_GRANULARITY_BYTES));
|
|
// Spin while the barrier's phase hasn't flipped yet. The report outputs
|
|
// are only valid once the barrier completes, so inspect them afterward.
|
|
bool isReportSeen = false;
|
|
uint8_t reportValue = 0;
|
|
bool complete = waitWithDebugTimeout(
|
|
[&]() {
|
|
return !cuda::ptx::mbarrier_try_wait_parity(cuda::ptx::mbarrier_phase_primary,
|
|
cuda::ptx::sem_relaxed,
|
|
cuda::ptx::scope_cta,
|
|
isReportSeen,
|
|
reportValue,
|
|
&putBar,
|
|
/*phaseParity=*/0);
|
|
},
|
|
"cftTryPutCountedKernel: mbarrier try_wait parity");
|
|
if (!complete) {
|
|
__trap();
|
|
}
|
|
if (isReportSeen) {
|
|
reportFabricError(&reportValue, offset);
|
|
}
|
|
}
|
|
// Block 0 waits until our receive byte counter reaches expectedBytes. System
|
|
// scope is required because the peer's counted fabric puts update this
|
|
// counter in our fabric-bound memory, not via threads in this CTA.
|
|
// A relaxed load suffices: the fabric-proxy is a strong proxy in the PTX
|
|
// memory model, so fabric-delivered writes are eventually observable
|
|
if (blockIdx.x == 0 && threadIdx.x == 0) {
|
|
cuda::atomic_ref<uint64_t, cuda::thread_scope_system> ref(*counterPtr);
|
|
if (!waitWithDebugTimeout([&]() { return ref.load(cuda::memory_order_relaxed) != expectedBytes; },
|
|
"cftTryPutCountedKernel: receive byte counter")) {
|
|
__trap();
|
|
}
|
|
}
|
|
}
|
|
|
|
// Non-counted variant: each block sends one FABRIC_CHUNK_SIZE chunk via plain
|
|
// fabric.try_put (no byte counter). This kernel only moves payload; the inter-GPU
|
|
// completion handshake happens in cftSignalFlagKernel, launched next on the same
|
|
// stream. The kernel boundary is the cross-block sync point -- once this kernel
|
|
// returns, every block's put has been issued and its source SMEM consumed.
|
|
__global__ void cftTryPutKernel(uint32_t putLeId, int myRank)
|
|
{
|
|
constexpr int elems = FABRIC_CHUNK_SIZE / sizeof(int);
|
|
__shared__ alignas(16) int smemBuf[elems];
|
|
__shared__ __mbarrier_t putBar;
|
|
|
|
if (threadIdx.x == 0) {
|
|
cuda::ptx::mbarrier_init(cuda::ptx::layout_v1, &putBar, 1);
|
|
}
|
|
|
|
for (int i = threadIdx.x; i < elems; i += blockDim.x) {
|
|
smemBuf[i] = myRank * elems + i;
|
|
}
|
|
__syncthreads();
|
|
|
|
if (threadIdx.x == 0) {
|
|
const uint64_t offset = blockIdx.x * FABRIC_CHUNK_SIZE;
|
|
// Make the staged shared-memory source visible to the async proxy before the put reads it.
|
|
cuda::ptx::fence_proxy_async(cuda::ptx::space_shared);
|
|
cuda::ptx::fabric_try_put(cuda::ptx::space_shared,
|
|
cuda::ptx::sem_relaxed,
|
|
cuda::ptx::scope_sys,
|
|
putLeId,
|
|
offset,
|
|
smemBuf,
|
|
FABRIC_CHUNK_SIZE,
|
|
&putBar);
|
|
cuda::ptx::fabric_submit();
|
|
cuda::ptx::mbarrier_arrive_expect_tx(cuda::ptx::sem_relaxed,
|
|
cuda::ptx::scope_cta,
|
|
cuda::ptx::space_shared,
|
|
&putBar,
|
|
(FABRIC_CHUNK_SIZE / FABRIC_TX_GRANULARITY_BYTES));
|
|
// Spin while the barrier's phase hasn't flipped yet. The report outputs
|
|
// are only valid once the barrier completes, so inspect them afterward.
|
|
bool isReportSeen = false;
|
|
uint8_t reportValue = 0;
|
|
bool complete = waitWithDebugTimeout(
|
|
[&]() {
|
|
return !cuda::ptx::mbarrier_try_wait_parity(cuda::ptx::mbarrier_phase_primary,
|
|
cuda::ptx::sem_relaxed,
|
|
cuda::ptx::scope_cta,
|
|
isReportSeen,
|
|
reportValue,
|
|
&putBar,
|
|
/*phaseParity=*/0);
|
|
},
|
|
"cftTryPutKernel: mbarrier try_wait parity");
|
|
if (!complete) {
|
|
__trap();
|
|
}
|
|
if (isReportSeen) {
|
|
reportFabricError(&reportValue, offset);
|
|
}
|
|
}
|
|
}
|
|
|
|
// Inter-GPU completion signal for the default (non-counted) path, launched with one
|
|
// after cftTryPutKernel. Because it is ordered on the same stream, the payload
|
|
// kernel has already finished, so we issue a single flag put to signal completion.
|
|
//
|
|
// Note: this kernel could be overlapped with the tail of cftTryPutKernel using
|
|
// Programmatic Dependent Launch (PDL) to hide its launch latency. The prologue
|
|
// (barrier init, staging smemFlag) may run early, but the flag put must still be
|
|
// gated on payload completion via cudaGridDependencySynchronize(), so we keep the
|
|
// kernels plainly stream-ordered here for clarity.
|
|
__global__ void cftSignalFlagKernel(uint32_t putLeId, uint64_t interGpuFlagOffset, uint32_t *interGpuFlag)
|
|
{
|
|
__shared__ alignas(16) uint32_t smemFlag[INTER_GPU_FLAG_BYTES / sizeof(uint32_t)];
|
|
__shared__ __mbarrier_t putBar;
|
|
|
|
// This kernel is a single-thread, single-block signal: it is launched <<<1, 1>>>
|
|
// and its one thread issues the flag put and waits on the neighbor's flag.
|
|
assert(blockDim.x == 1 && gridDim.x == 1);
|
|
|
|
cuda::ptx::mbarrier_init(cuda::ptx::layout_v1, &putBar, 1);
|
|
// A b128 fabric put moves 16 bytes. Initialize the full source even though
|
|
// the receiver treats a non-zero first word as the signal.
|
|
for (size_t i = 0; i < INTER_GPU_FLAG_BYTES / sizeof(uint32_t); ++i) {
|
|
smemFlag[i] = 1;
|
|
}
|
|
cuda::ptx::fence_proxy_async(cuda::ptx::space_shared);
|
|
cuda::ptx::fabric_try_put(cuda::ptx::space_shared,
|
|
cuda::ptx::sem_relaxed,
|
|
cuda::ptx::scope_sys,
|
|
putLeId,
|
|
interGpuFlagOffset,
|
|
smemFlag,
|
|
INTER_GPU_FLAG_BYTES,
|
|
&putBar);
|
|
cuda::ptx::fabric_submit();
|
|
cuda::ptx::mbarrier_arrive_expect_tx(cuda::ptx::sem_relaxed,
|
|
cuda::ptx::scope_cta,
|
|
cuda::ptx::space_shared,
|
|
&putBar,
|
|
(INTER_GPU_FLAG_BYTES / FABRIC_TX_GRANULARITY_BYTES));
|
|
// Spin while the barrier's phase hasn't flipped yet. The report outputs are
|
|
// only valid once the barrier completes, so inspect them afterward.
|
|
bool isReportSeen = false;
|
|
uint8_t reportValue = 0;
|
|
bool complete = waitWithDebugTimeout(
|
|
[&]() {
|
|
return !cuda::ptx::mbarrier_try_wait_parity(cuda::ptx::mbarrier_phase_primary,
|
|
cuda::ptx::sem_relaxed,
|
|
cuda::ptx::scope_cta,
|
|
isReportSeen,
|
|
reportValue,
|
|
&putBar,
|
|
/*phaseParity=*/0);
|
|
},
|
|
"cftSignalFlagKernel: mbarrier try_wait parity");
|
|
if (!complete) {
|
|
__trap();
|
|
}
|
|
if (isReportSeen) {
|
|
reportFabricError(&reportValue, interGpuFlagOffset);
|
|
}
|
|
|
|
// Wait for the receive neighbor to set our flag via a fabric put.
|
|
cuda::atomic_ref<uint32_t, cuda::thread_scope_system> ref(*interGpuFlag);
|
|
if (!waitWithDebugTimeout([&]() { return ref.load(cuda::memory_order_relaxed) == 0; },
|
|
"cftSignalFlagKernel: inter-GPU flag")) {
|
|
__trap();
|
|
}
|
|
}
|
|
|
|
// Confirm the device exposes the capabilities this example relies on: fabric
|
|
// handles are always required; counted logical-endpoint ops only in --counted mode.
|
|
// Returns false when the sample is not supported on this device (caller should
|
|
// exit with EXIT_WAIVED, not EXIT_FAILURE).
|
|
static bool requireDeviceCapabilities(CUdevice dev, bool needCounted)
|
|
{
|
|
int vmmSupported = 0;
|
|
checkCudaErrors(cuDeviceGetAttribute(&vmmSupported, CU_DEVICE_ATTRIBUTE_VIRTUAL_MEMORY_MANAGEMENT_SUPPORTED, dev));
|
|
if (!vmmSupported) {
|
|
fprintf(stderr, "device does not support virtual memory management\n");
|
|
return false;
|
|
}
|
|
|
|
int unicastSupported = 0;
|
|
checkCudaErrors(cuDeviceGetAttribute(&unicastSupported, CU_DEVICE_ATTRIBUTE_LOGICAL_ENDPOINT_UNICAST_SUPPORTED, dev));
|
|
if (!unicastSupported) {
|
|
fprintf(stderr, "device does not support unicast logical endpoints\n");
|
|
return false;
|
|
}
|
|
|
|
int fabricSupported = 0;
|
|
checkCudaErrors(cuDeviceGetAttribute(&fabricSupported, CU_DEVICE_ATTRIBUTE_HANDLE_TYPE_FABRIC_SUPPORTED, dev));
|
|
if (!fabricSupported) {
|
|
fprintf(stderr, "device does not support fabric handles\n");
|
|
return false;
|
|
}
|
|
|
|
// Bitmask of CUlogicalEndpointIpcHandleType.
|
|
int leHandleTypes = 0;
|
|
checkCudaErrors(cuDeviceGetAttribute(&leHandleTypes, CU_DEVICE_ATTRIBUTE_LOGICAL_ENDPOINT_SUPPORTED_HANDLE_TYPES, dev));
|
|
if ((leHandleTypes & CU_LOGICAL_ENDPOINT_IPC_HANDLE_TYPE_FABRIC) == 0) {
|
|
fprintf(stderr, "device does not support fabric IPC for logical endpoints\n");
|
|
return false;
|
|
}
|
|
|
|
if (needCounted) {
|
|
int countedSupported = 0;
|
|
checkCudaErrors(
|
|
cuDeviceGetAttribute(&countedSupported, CU_DEVICE_ATTRIBUTE_LOGICAL_ENDPOINT_COUNTED_OPS_SUPPORTED, dev));
|
|
if (!countedSupported) {
|
|
fprintf(stderr, "device does not support counted logical-endpoint ops\n");
|
|
return false;
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
|
|
static constexpr int DEFAULT_CHUNK_CNT = 100;
|
|
|
|
static void printUsage(const char *prog)
|
|
{
|
|
fprintf(stderr,
|
|
"Usage: mpirun -np <N> %s [chunkCnt] [--counted]\n"
|
|
" <N> number of MPI ranks (must be >= 2)\n"
|
|
" [chunkCnt] number of %u-byte chunks each rank sends (default: %d)\n"
|
|
" --counted use counted fabric puts with a byte-arrival counter\n"
|
|
"Example: mpirun -np 2 %s\n",
|
|
prog,
|
|
FABRIC_CHUNK_SIZE,
|
|
DEFAULT_CHUNK_CNT,
|
|
prog);
|
|
}
|
|
|
|
int main(int argc, char *argv[])
|
|
{
|
|
int chunkCnt = DEFAULT_CHUNK_CNT;
|
|
bool useCounted = false;
|
|
for (int i = 1; i < argc; ++i) {
|
|
if (strcmp(argv[i], "--counted") == 0) {
|
|
useCounted = true;
|
|
}
|
|
else if (strcmp(argv[i], "--help") == 0 || strcmp(argv[i], "-h") == 0) {
|
|
printUsage(argv[0]);
|
|
return EXIT_SUCCESS;
|
|
}
|
|
else {
|
|
char *end = nullptr;
|
|
const long parsed = strtol(argv[i], &end, 10);
|
|
if (end == argv[i] || *end != '\0' || parsed <= 0 || parsed > INT_MAX) {
|
|
fprintf(stderr, "error: invalid argument '%s'\n", argv[i]);
|
|
printUsage(argv[0]);
|
|
return EXIT_FAILURE;
|
|
}
|
|
chunkCnt = static_cast<int>(parsed);
|
|
}
|
|
}
|
|
const size_t totalBytes = static_cast<size_t>(FABRIC_CHUNK_SIZE) * chunkCnt;
|
|
|
|
// ---- MPI setup: ring topology, rank i sends to (i+1)%N. ----
|
|
int myRank = 0;
|
|
int numRanks = 0;
|
|
MPI_CHECK(MPI_Init(&argc, &argv));
|
|
MPI_CHECK(MPI_Comm_rank(MPI_COMM_WORLD, &myRank));
|
|
MPI_CHECK(MPI_Comm_size(MPI_COMM_WORLD, &numRanks));
|
|
if (numRanks < 2) {
|
|
if (myRank == 0) {
|
|
fprintf(stderr, "error: this example needs at least 2 MPI ranks (got %d)\n", numRanks);
|
|
}
|
|
MPI_CHECK(MPI_Finalize());
|
|
return EXIT_FAILURE;
|
|
}
|
|
if (myRank == 0) {
|
|
printf("mode = %s, chunkCnt = %d, totalBytes = %zu, ranks = %d\n",
|
|
useCounted ? "counted" : "put",
|
|
chunkCnt,
|
|
totalBytes,
|
|
numRanks);
|
|
}
|
|
// Ring neighbors: put to sendRank; receive completion from recvRank.
|
|
const int sendRank = (myRank + 1) % numRanks;
|
|
const int recvRank = (myRank + numRanks - 1) % numRanks;
|
|
|
|
// ---- Device context and stream. ----
|
|
// Device ordinals are node-local, so select a device using this process's
|
|
// rank among the MPI processes on the same node.
|
|
MPI_Comm localComm = MPI_COMM_NULL;
|
|
MPI_CHECK(MPI_Comm_split_type(MPI_COMM_WORLD, MPI_COMM_TYPE_SHARED, myRank, MPI_INFO_NULL, &localComm));
|
|
int localRank = 0;
|
|
MPI_CHECK(MPI_Comm_rank(localComm, &localRank));
|
|
|
|
int deviceCount = 0;
|
|
checkCudaErrors(cudaGetDeviceCount(&deviceCount));
|
|
if (localRank >= deviceCount) {
|
|
fprintf(stderr,
|
|
"error: local MPI rank %d cannot be assigned a unique device; "
|
|
"only %d device(s) are visible on this node\n",
|
|
localRank,
|
|
deviceCount);
|
|
MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE);
|
|
return EXIT_FAILURE;
|
|
}
|
|
const int deviceOrdinal = localRank;
|
|
|
|
checkCudaErrors(cudaSetDevice(deviceOrdinal));
|
|
CUdevice cuDevice;
|
|
checkCudaErrors(cuInit(0));
|
|
checkCudaErrors(cuDeviceGet(&cuDevice, deviceOrdinal));
|
|
int localSupported = requireDeviceCapabilities(cuDevice, useCounted) ? 1 : 0;
|
|
int allSupported = 0;
|
|
MPI_CHECK(MPI_Allreduce(&localSupported, &allSupported, 1, MPI_INT, MPI_MIN, MPI_COMM_WORLD));
|
|
if (!allSupported) {
|
|
MPI_CHECK(MPI_Comm_free(&localComm));
|
|
MPI_CHECK(MPI_Finalize());
|
|
return EXIT_WAIVED;
|
|
}
|
|
CUstream stream;
|
|
checkCudaErrors(cuStreamCreate(&stream, CU_STREAM_NON_BLOCKING));
|
|
|
|
// Query how many cliques this device is a member of:
|
|
size_t cliqueCount = 0; // not modified if there are errors
|
|
checkCudaErrors(cuDeviceGetCliqueCount(&cliqueCount, cuDevice));
|
|
if (cliqueCount == 0) {
|
|
fprintf(stderr, "This GPU is not a member of any clique.");
|
|
}
|
|
|
|
// Query the clique info for all cliques
|
|
std::vector<CUcliqueInfo> cliqueInfos(cliqueCount);
|
|
if (cliqueCount > 0) {
|
|
checkCudaErrors(cuDeviceGetCliqueInfo(cliqueInfos.data(), &cliqueCount, cuDevice));
|
|
}
|
|
unsigned int localCliqueId = 0;
|
|
bool found = false;
|
|
for (size_t idx = 0; idx < cliqueCount; ++idx) {
|
|
if (cliqueInfos[idx].type == CU_CLIQUE_TYPE_UNICAST_LOGICAL_ENDPOINT) {
|
|
localCliqueId = cliqueInfos[idx].id;
|
|
found = true;
|
|
break;
|
|
}
|
|
}
|
|
if (cliqueCount != 0 && !found) {
|
|
fprintf(stderr, "This GPU is not a member of a clique with unicast logical endpoint support.");
|
|
}
|
|
int localCliqueOk = found ? 1 : 0;
|
|
int allCliqueOk = 0;
|
|
MPI_CHECK(MPI_Allreduce(&localCliqueOk, &allCliqueOk, 1, MPI_INT, MPI_MIN, MPI_COMM_WORLD));
|
|
if (!allCliqueOk) {
|
|
checkCudaErrors(cuStreamDestroy(stream));
|
|
MPI_CHECK(MPI_Comm_free(&localComm));
|
|
MPI_CHECK(MPI_Finalize());
|
|
return EXIT_WAIVED;
|
|
}
|
|
|
|
// Verify that all MPI ranks found the same clique id:
|
|
std::vector<unsigned int> allCliqueIds(numRanks);
|
|
MPI_CHECK(MPI_Allgather(&localCliqueId,
|
|
sizeof(unsigned int),
|
|
MPI_BYTE,
|
|
allCliqueIds.data(),
|
|
sizeof(unsigned int),
|
|
MPI_BYTE,
|
|
MPI_COMM_WORLD));
|
|
for (int i = 0; i < numRanks; i++) {
|
|
if (allCliqueIds[i] != localCliqueId) {
|
|
fprintf(stderr,
|
|
"Rank %d: Clique id %u does not match expected %u\n",
|
|
myRank,
|
|
allCliqueIds[i],
|
|
localCliqueId);
|
|
return EXIT_FAILURE;
|
|
}
|
|
}
|
|
|
|
// Reserve enough local ids for this rank's endpoint and all imported peers.
|
|
CUlogicalEndpointId leId = 0;
|
|
checkCudaErrors(cuLogicalEndpointIdReserve(&leId, (uint32_t)numRanks));
|
|
|
|
// ---- Describe the fabric-shareable allocation and query its granularity. ----
|
|
CUmemAllocationProp exportAllocProp = {};
|
|
exportAllocProp.type = CU_MEM_ALLOCATION_TYPE_PINNED;
|
|
exportAllocProp.location.type = CU_MEM_LOCATION_TYPE_DEVICE;
|
|
exportAllocProp.location.id = cuDevice;
|
|
exportAllocProp.requestedHandleTypes = CU_MEM_HANDLE_TYPE_FABRIC;
|
|
size_t granularity = 0;
|
|
checkCudaErrors(cuMemGetAllocationGranularity(&granularity, &exportAllocProp, CU_MEM_ALLOC_GRANULARITY_RECOMMENDED));
|
|
|
|
// ---- Describe the logical endpoint (unicast; counted ops optional). ----
|
|
CUlogicalEndpointProp endpointProp{};
|
|
endpointProp.type = CU_LOGICAL_ENDPOINT_TYPE_UNICAST;
|
|
endpointProp.unicast = {.device = cuDevice};
|
|
endpointProp.ipcHandleTypes = CU_LOGICAL_ENDPOINT_IPC_HANDLE_TYPE_FABRIC;
|
|
endpointProp.flags = useCounted ? CU_LOGICAL_ENDPOINT_FLAG_COUNTED_OPS : CU_LOGICAL_ENDPOINT_FLAG_NONE;
|
|
cuuint64_t bindAlignment;
|
|
cuuint64_t maxSize;
|
|
checkCudaErrors(cuLogicalEndpointGetLimits(&bindAlignment, &maxSize, &endpointProp));
|
|
// Make the allocation and endpoint size a multiple of both independent
|
|
// alignment requirements.
|
|
const size_t requiredAlignment = std::lcm(granularity, static_cast<size_t>(bindAlignment));
|
|
|
|
// ---- Allocate payload, sync region, and alignment padding. ----
|
|
size_t exportSize = alignUp(totalBytes + SYNC_REGION_ALIGN_BYTES + SYNC_REGION_BYTES, requiredAlignment);
|
|
if (exportSize > maxSize) {
|
|
fprintf(stderr,
|
|
"error: required endpoint size %zu exceeds maximum %llu\n",
|
|
exportSize,
|
|
static_cast<unsigned long long>(maxSize));
|
|
MPI_CHECK(MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE));
|
|
return EXIT_FAILURE;
|
|
}
|
|
endpointProp.size = exportSize;
|
|
|
|
// ---- Create the local endpoint, then exchange handles with send neighbor. ----
|
|
checkCudaErrors(cuLogicalEndpointCreate(leId + myRank, &endpointProp));
|
|
// Export our own endpoint to a fabric IPC handle, hand it to the peers that
|
|
// need it, and import each peer's handle to a locally reserved id.
|
|
CUlogicalEndpointFabricHandle localHandle;
|
|
CUlogicalEndpointFabricHandle sendRankHandle;
|
|
checkCudaErrors(cuLogicalEndpointExport(&localHandle, leId + myRank, CU_LOGICAL_ENDPOINT_IPC_HANDLE_TYPE_FABRIC));
|
|
MPI_CHECK(MPI_Sendrecv(&localHandle,
|
|
sizeof(CUlogicalEndpointFabricHandle),
|
|
MPI_BYTE,
|
|
recvRank,
|
|
0,
|
|
&sendRankHandle,
|
|
sizeof(CUlogicalEndpointFabricHandle),
|
|
MPI_BYTE,
|
|
sendRank,
|
|
0,
|
|
MPI_COMM_WORLD,
|
|
MPI_STATUS_IGNORE));
|
|
checkCudaErrors(cuLogicalEndpointImport(leId + sendRank, &sendRankHandle, CU_LOGICAL_ENDPOINT_IPC_HANDLE_TYPE_FABRIC));
|
|
// Create and import are non-blocking, so wait until both endpoints are ready.
|
|
waitForEndpointReady(leId + myRank);
|
|
waitForEndpointReady(leId + sendRank);
|
|
|
|
// ---- Allocate and bind memory after sharing the endpoint. ----
|
|
CUmemGenericAllocationHandle exportHandle;
|
|
CUdeviceptr exportPtr = 0;
|
|
checkCudaErrors(cuMemCreate(&exportHandle, exportSize, &exportAllocProp, 0));
|
|
checkCudaErrors(cuMemAddressReserve(&exportPtr, exportSize, granularity, 0, 0));
|
|
checkCudaErrors(cuMemMap(exportPtr, exportSize, 0, exportHandle, 0));
|
|
CUmemAccessDesc accessDesc = {};
|
|
accessDesc.location.type = CU_MEM_LOCATION_TYPE_DEVICE;
|
|
accessDesc.location.id = exportAllocProp.location.id;
|
|
accessDesc.flags = CU_MEM_ACCESS_FLAGS_PROT_READWRITE;
|
|
checkCudaErrors(cuMemSetAccess(exportPtr, exportSize, &accessDesc, 1));
|
|
// Zero the whole allocation before the first transfer or barrier handshake.
|
|
checkCudaErrors(cuMemsetD8(exportPtr, 0, exportSize));
|
|
// 256-byte-aligned sync region after the payload (required for counted fabric puts).
|
|
std::byte *const basePtr = reinterpret_cast<std::byte *>(exportPtr);
|
|
void *syncPtr = basePtr + totalBytes;
|
|
size_t syncSpace = exportSize - totalBytes;
|
|
syncPtr = std::align(SYNC_REGION_ALIGN_BYTES, SYNC_REGION_BYTES, syncPtr, syncSpace);
|
|
assert(syncPtr != nullptr); // guaranteed by the SYNC_REGION_ALIGN_BYTES slack reserved above
|
|
const size_t syncRegionOffset = static_cast<std::byte *>(syncPtr) - basePtr;
|
|
// counted mode (--counted): local byte counter the peer's counted puts accumulate into.
|
|
uint64_t *const counterPtr = reinterpret_cast<uint64_t *>(syncPtr);
|
|
// default mode: local inter-GPU flag, set by the receive neighbor's fabric put.
|
|
uint32_t *const interGpuFlag = reinterpret_cast<uint32_t *>(syncPtr);
|
|
// Bind the whole backing allocation at endpoint offset 0, by allocation handle.
|
|
checkCudaErrors(cuLogicalEndpointBindMem(leId + myRank, cuDevice, 0, exportHandle, 0, exportSize, 0));
|
|
// No rank may issue an operation until every destination has been bound.
|
|
MPI_CHECK(MPI_Barrier(MPI_COMM_WORLD));
|
|
|
|
// ---- Push data to sendRank. The kernel writes into the neighbor's endpoint
|
|
// (leId + sendRank); the receive neighbor's puts land in our own buffer. ----
|
|
const uint32_t sendLe = leId + sendRank;
|
|
const uint64_t counterOffset = syncRegionOffset;
|
|
const uint64_t interGpuFlagOffset = syncRegionOffset;
|
|
|
|
// Launch one block per chunk: gridDim.x == totalBytes / FABRIC_CHUNK_SIZE.
|
|
if (useCounted) {
|
|
cftTryPutCountedKernel<<<chunkCnt, WARP_SIZE, 0, stream>>>(sendLe,
|
|
counterOffset,
|
|
myRank,
|
|
counterPtr,
|
|
/*expectedBytes=*/totalBytes);
|
|
}
|
|
else {
|
|
// Payload first, then a one-thread flag kernel; same-stream ordering guarantees
|
|
// the payload puts are done before the inter-GPU flag handshake runs.
|
|
cftTryPutKernel<<<chunkCnt, WARP_SIZE, 0, stream>>>(sendLe, myRank);
|
|
cftSignalFlagKernel<<<1, 1, 0, stream>>>(sendLe, interGpuFlagOffset, interGpuFlag);
|
|
}
|
|
checkCudaErrors(cuStreamSynchronize(stream));
|
|
|
|
// ---- Check the payload the receive neighbor pushed into our endpoint. ----
|
|
// Every sending block stages the same rank-tagged pattern, so the expected
|
|
// value at int index i is recvRank * elemsPerChunk + (i % elemsPerChunk).
|
|
constexpr size_t elemsPerChunk = FABRIC_CHUNK_SIZE / sizeof(int);
|
|
const size_t totalElems = totalBytes / sizeof(int);
|
|
std::vector<int> received(totalElems);
|
|
checkCudaErrors(cuMemcpyDtoH(received.data(), exportPtr, totalBytes));
|
|
|
|
size_t mismatches = 0;
|
|
for (size_t i = 0; i < totalElems; ++i) {
|
|
const int expected = static_cast<int>(recvRank * elemsPerChunk + i % elemsPerChunk);
|
|
if (received[i] != expected) {
|
|
if (mismatches == 0) {
|
|
fprintf(stderr,
|
|
"rank %d: first mismatch at int index %zu: got %d, expected %d\n",
|
|
myRank,
|
|
i,
|
|
received[i],
|
|
expected);
|
|
}
|
|
++mismatches;
|
|
}
|
|
}
|
|
|
|
int localFailed = (mismatches != 0) ? 1 : 0;
|
|
int anyFailed = 0;
|
|
MPI_CHECK(MPI_Allreduce(&localFailed, &anyFailed, 1, MPI_INT, MPI_MAX, MPI_COMM_WORLD));
|
|
if (myRank == 0) {
|
|
printf("Result = %s\n", anyFailed ? "FAIL" : "PASS");
|
|
}
|
|
|
|
// ---- Clean up endpoints and free the backing allocation. ----
|
|
// Once every rank's stream has synchronized, all fabric puts
|
|
// have completed and no rank will issue another operation. Making that global
|
|
// with a barrier ensures no peer still targets an endpoint we clean up below.
|
|
MPI_CHECK(MPI_Barrier(MPI_COMM_WORLD));
|
|
|
|
// Destroy the imported endpoint (the peer target our puts addressed). An import
|
|
// is only a local reference to the peer's endpoint id and maps no peer memory,
|
|
// so once our puts have drained this just drops that reference.
|
|
checkCudaErrors(cuLogicalEndpointDestroy(leId + sendRank));
|
|
|
|
// Unbind and destroy our owned endpoint, then release the reserved id range.
|
|
checkCudaErrors(cuLogicalEndpointUnbind(leId + myRank, cuDevice, 0, exportSize));
|
|
checkCudaErrors(cuLogicalEndpointDestroy(leId + myRank));
|
|
checkCudaErrors(cuLogicalEndpointIdRelease(leId, numRanks));
|
|
|
|
// Free the backing allocation in VMM cleanup order: unmap the range, release
|
|
// the physical handle, then free the reserved virtual address range.
|
|
checkCudaErrors(cuMemUnmap(exportPtr, exportSize));
|
|
checkCudaErrors(cuMemRelease(exportHandle));
|
|
checkCudaErrors(cuMemAddressFree(exportPtr, exportSize));
|
|
|
|
checkCudaErrors(cuStreamDestroy(stream));
|
|
MPI_CHECK(MPI_Comm_free(&localComm));
|
|
MPI_CHECK(MPI_Finalize());
|
|
|
|
return anyFailed ? EXIT_FAILURE : EXIT_SUCCESS;
|
|
}
|