/* Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * Neither the name of NVIDIA CORPORATION nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /* * Unicast point-to-point data movement with Compute Fabric Transport (CFT) * logical endpoints. * * MPI example on a ring: rank r sends to (r+1)%N and receives from (r-1+N)%N. * Default mode uses plain fabric puts; a separate one-thread kernel then flags the * send neighbor over the same endpoint and waits on the receive neighbor's flag. * Pass --counted for counted fabric puts and a counter which counts bytes arrival * for receive completion. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifndef EXIT_WAIVED #define EXIT_WAIVED 2 #endif #define MPI_CHECK(cmd) \ do { \ int e = cmd; \ if (e != MPI_SUCCESS) { \ printf("Failed: MPI error %s:%d '%d'\n", __FILE__, __LINE__, e); \ exit(EXIT_FAILURE); \ } \ } while (0) static void checkCuda(CUresult result, const char *func, const char *file, int line) { if (result != CUDA_SUCCESS) { const char *name = nullptr; cuGetErrorName(result, &name); fprintf(stderr, "CUDA driver error at %s:%d code=%d(%s) \"%s\"\n", file, line, static_cast(result), name ? name : "", func); exit(EXIT_FAILURE); } } static void checkCuda(cudaError_t result, const char *func, const char *file, int line) { if (result != cudaSuccess) { fprintf(stderr, "CUDA runtime error at %s:%d code=%d(%s) \"%s\"\n", file, line, static_cast(result), cudaGetErrorName(result), func); exit(EXIT_FAILURE); } } #define checkCudaErrors(val) checkCuda((val), #val, __FILE__, __LINE__) // Round value up to the next multiple of alignment (alignment need not be a power of two). constexpr size_t alignUp(size_t value, size_t alignment) { return (value + alignment - 1) / alignment * alignment; } void waitForEndpointReady(CUlogicalEndpointId leId) { while (true) { int ready = 0; checkCudaErrors(cuLogicalEndpointQuery(leId, 1, &ready)); if (ready > 0) break; } } static constexpr uint32_t FABRIC_CHUNK_SIZE = 8192; // A b128 fabric put moves 16 bytes at a time, and mbarrier complete_tx::16B // counts transferred bytes in 16-byte units. So an expected-tx count is always // the byte count divided by this granularity. static constexpr uint32_t FABRIC_TX_GRANULARITY_BYTES = 16; // Sync region placed after the payload on a 256-byte boundary, reused per mode: // default : a 16-byte inter-GPU flag that the receive neighbor sets with a // fabric put (b128 puts move 16 bytes at a time). // --counted : an 8-byte delivered-byte counter. static constexpr size_t INTER_GPU_FLAG_BYTES = 16; static constexpr size_t SYNC_REGION_BYTES = 16; // max of counter (8B) and flag (16B) // The counter targeted by counted fabric puts must be 256-byte aligned, so the // whole sync region is placed on a 256-byte boundary. static constexpr size_t SYNC_REGION_ALIGN_BYTES = 256; static constexpr int WARP_SIZE = 32; #ifndef NDEBUG // Debug builds: cap spin-waits at 30 s so a stuck fabric op prints instead of hanging. static constexpr uint64_t CFT_WAIT_TIMEOUT_NS = 30ULL * 1000 * 1000 * 1000; inline __device__ uint64_t readGlobalTimerNs() { uint64_t t = 0; asm volatile("mov.u64 %0, %%globaltimer;" : "=l"(t)); return t; } #endif // Spin while pred() is true. Returns true when pred() becomes false; in debug // builds returns false after CFT_WAIT_TIMEOUT_NS and prints `what` to identify // the stall. Release builds spin forever. Caller must handle false (e.g. __trap()). template inline __device__ bool waitWithDebugTimeout(Predicate &&pred, const char *what) { #ifndef NDEBUG uint64_t start = readGlobalTimerNs(); while (pred()) { uint64_t now = readGlobalTimerNs(); if (now - start > CFT_WAIT_TIMEOUT_NS) { printf("%s timeout after %llu ns (block=%u)\n", what, (unsigned long long)(now - start), blockIdx.x); return false; } } #else (void)what; while (pred()) { } #endif return true; } // Decode and print the fabric errors captured in the mbarrier's report value. // Decode it with cudaFabricOpStatusSourceMbarrierV1: first query the error count, then // retrieve each decoded error. Call this only when the report predicate is set. inline __device__ void reportFabricError(uint8_t *reportValue, uint64_t offset) { const unsigned long long off = offset; unsigned int errCount = 0; if (cudaFabricOpErrorStatusCount(reportValue, cudaFabricOpStatusSourceMbarrierV1, &errCount) != cudaSuccess) { printf("cudaFabricOpErrorStatusCount failed at offset %llu\n", off); return; } for (unsigned int i = 0; i < errCount; ++i) { cudaFabricOpStatusInfo info = cudaFabricOpStatusInfoSuccess; if (cudaFabricOpErrorStatusGet(reportValue, cudaFabricOpStatusSourceMbarrierV1, i, &info) == cudaSuccess) { printf("fabric put error[%u/%u] at offset %llu: status=%d\n", i, errCount, off, (int)info); } else { printf("cudaFabricOpErrorStatusGet failed for error[%u/%u] at offset %llu\n", i, errCount, off); } } } // Each block sends one FABRIC_CHUNK_SIZE slice to the peer's logical endpoint (putLeId). // blockIdx.x selects the chunk: offset = blockIdx.x * FABRIC_CHUNK_SIZE. Launch // gridDim.x = totalBytes / FABRIC_CHUNK_SIZE so every chunk is covered by one block. // // expectedBytes is how many bytes this rank expects to receive (tracked in our // local counter by the peer's puts). __global__ void cftTryPutCountedKernel(uint32_t putLeId, uint64_t counterOffset, int myRank, uint64_t *counterPtr, uint64_t expectedBytes) { constexpr int elems = FABRIC_CHUNK_SIZE / sizeof(int); __shared__ alignas(16) int smemBuf[elems]; __shared__ __mbarrier_t putBar; if (threadIdx.x == 0) { cuda::ptx::mbarrier_init(cuda::ptx::layout_v1, &putBar, 1); } // Fill shared memory with a rank-tagged pattern before streaming it to the peer. for (int i = threadIdx.x; i < elems; i += blockDim.x) { smemBuf[i] = myRank * elems + i; } __syncthreads(); // Thread 0 issues one counted fabric put for this block's chunk. if (threadIdx.x == 0) { const uint64_t offset = blockIdx.x * FABRIC_CHUNK_SIZE; // Make the staged shared-memory source visible to the async proxy before the put reads it. cuda::ptx::fence_proxy_async(cuda::ptx::space_shared); cuda::ptx::fabric_try_put_counted(cuda::ptx::space_shared, cuda::ptx::sem_relaxed, cuda::ptx::scope_sys, putLeId, offset, counterOffset, smemBuf, FABRIC_CHUNK_SIZE, &putBar); cuda::ptx::fabric_submit(); cuda::ptx::mbarrier_arrive_expect_tx(cuda::ptx::sem_relaxed, cuda::ptx::scope_cta, cuda::ptx::space_shared, &putBar, (FABRIC_CHUNK_SIZE / FABRIC_TX_GRANULARITY_BYTES)); // Spin while the barrier's phase hasn't flipped yet. The report outputs // are only valid once the barrier completes, so inspect them afterward. bool isReportSeen = false; uint8_t reportValue = 0; bool complete = waitWithDebugTimeout( [&]() { return !cuda::ptx::mbarrier_try_wait_parity(cuda::ptx::mbarrier_phase_primary, cuda::ptx::sem_relaxed, cuda::ptx::scope_cta, isReportSeen, reportValue, &putBar, /*phaseParity=*/0); }, "cftTryPutCountedKernel: mbarrier try_wait parity"); if (!complete) { __trap(); } if (isReportSeen) { reportFabricError(&reportValue, offset); } } // Block 0 waits until our receive byte counter reaches expectedBytes. System // scope is required because the peer's counted fabric puts update this // counter in our fabric-bound memory, not via threads in this CTA. // A relaxed load suffices: the fabric-proxy is a strong proxy in the PTX // memory model, so fabric-delivered writes are eventually observable if (blockIdx.x == 0 && threadIdx.x == 0) { cuda::atomic_ref ref(*counterPtr); if (!waitWithDebugTimeout([&]() { return ref.load(cuda::memory_order_relaxed) != expectedBytes; }, "cftTryPutCountedKernel: receive byte counter")) { __trap(); } } } // Non-counted variant: each block sends one FABRIC_CHUNK_SIZE chunk via plain // fabric.try_put (no byte counter). This kernel only moves payload; the inter-GPU // completion handshake happens in cftSignalFlagKernel, launched next on the same // stream. The kernel boundary is the cross-block sync point -- once this kernel // returns, every block's put has been issued and its source SMEM consumed. __global__ void cftTryPutKernel(uint32_t putLeId, int myRank) { constexpr int elems = FABRIC_CHUNK_SIZE / sizeof(int); __shared__ alignas(16) int smemBuf[elems]; __shared__ __mbarrier_t putBar; if (threadIdx.x == 0) { cuda::ptx::mbarrier_init(cuda::ptx::layout_v1, &putBar, 1); } for (int i = threadIdx.x; i < elems; i += blockDim.x) { smemBuf[i] = myRank * elems + i; } __syncthreads(); if (threadIdx.x == 0) { const uint64_t offset = blockIdx.x * FABRIC_CHUNK_SIZE; // Make the staged shared-memory source visible to the async proxy before the put reads it. cuda::ptx::fence_proxy_async(cuda::ptx::space_shared); cuda::ptx::fabric_try_put(cuda::ptx::space_shared, cuda::ptx::sem_relaxed, cuda::ptx::scope_sys, putLeId, offset, smemBuf, FABRIC_CHUNK_SIZE, &putBar); cuda::ptx::fabric_submit(); cuda::ptx::mbarrier_arrive_expect_tx(cuda::ptx::sem_relaxed, cuda::ptx::scope_cta, cuda::ptx::space_shared, &putBar, (FABRIC_CHUNK_SIZE / FABRIC_TX_GRANULARITY_BYTES)); // Spin while the barrier's phase hasn't flipped yet. The report outputs // are only valid once the barrier completes, so inspect them afterward. bool isReportSeen = false; uint8_t reportValue = 0; bool complete = waitWithDebugTimeout( [&]() { return !cuda::ptx::mbarrier_try_wait_parity(cuda::ptx::mbarrier_phase_primary, cuda::ptx::sem_relaxed, cuda::ptx::scope_cta, isReportSeen, reportValue, &putBar, /*phaseParity=*/0); }, "cftTryPutKernel: mbarrier try_wait parity"); if (!complete) { __trap(); } if (isReportSeen) { reportFabricError(&reportValue, offset); } } } // Inter-GPU completion signal for the default (non-counted) path, launched with one // after cftTryPutKernel. Because it is ordered on the same stream, the payload // kernel has already finished, so we issue a single flag put to signal completion. // // Note: this kernel could be overlapped with the tail of cftTryPutKernel using // Programmatic Dependent Launch (PDL) to hide its launch latency. The prologue // (barrier init, staging smemFlag) may run early, but the flag put must still be // gated on payload completion via cudaGridDependencySynchronize(), so we keep the // kernels plainly stream-ordered here for clarity. __global__ void cftSignalFlagKernel(uint32_t putLeId, uint64_t interGpuFlagOffset, uint32_t *interGpuFlag) { __shared__ alignas(16) uint32_t smemFlag[INTER_GPU_FLAG_BYTES / sizeof(uint32_t)]; __shared__ __mbarrier_t putBar; // This kernel is a single-thread, single-block signal: it is launched <<<1, 1>>> // and its one thread issues the flag put and waits on the neighbor's flag. assert(blockDim.x == 1 && gridDim.x == 1); cuda::ptx::mbarrier_init(cuda::ptx::layout_v1, &putBar, 1); // A b128 fabric put moves 16 bytes. Initialize the full source even though // the receiver treats a non-zero first word as the signal. for (size_t i = 0; i < INTER_GPU_FLAG_BYTES / sizeof(uint32_t); ++i) { smemFlag[i] = 1; } cuda::ptx::fence_proxy_async(cuda::ptx::space_shared); cuda::ptx::fabric_try_put(cuda::ptx::space_shared, cuda::ptx::sem_relaxed, cuda::ptx::scope_sys, putLeId, interGpuFlagOffset, smemFlag, INTER_GPU_FLAG_BYTES, &putBar); cuda::ptx::fabric_submit(); cuda::ptx::mbarrier_arrive_expect_tx(cuda::ptx::sem_relaxed, cuda::ptx::scope_cta, cuda::ptx::space_shared, &putBar, (INTER_GPU_FLAG_BYTES / FABRIC_TX_GRANULARITY_BYTES)); // Spin while the barrier's phase hasn't flipped yet. The report outputs are // only valid once the barrier completes, so inspect them afterward. bool isReportSeen = false; uint8_t reportValue = 0; bool complete = waitWithDebugTimeout( [&]() { return !cuda::ptx::mbarrier_try_wait_parity(cuda::ptx::mbarrier_phase_primary, cuda::ptx::sem_relaxed, cuda::ptx::scope_cta, isReportSeen, reportValue, &putBar, /*phaseParity=*/0); }, "cftSignalFlagKernel: mbarrier try_wait parity"); if (!complete) { __trap(); } if (isReportSeen) { reportFabricError(&reportValue, interGpuFlagOffset); } // Wait for the receive neighbor to set our flag via a fabric put. cuda::atomic_ref ref(*interGpuFlag); if (!waitWithDebugTimeout([&]() { return ref.load(cuda::memory_order_relaxed) == 0; }, "cftSignalFlagKernel: inter-GPU flag")) { __trap(); } } // Confirm the device exposes the capabilities this example relies on: fabric // handles are always required; counted logical-endpoint ops only in --counted mode. // Returns false when the sample is not supported on this device (caller should // exit with EXIT_WAIVED, not EXIT_FAILURE). static bool requireDeviceCapabilities(CUdevice dev, bool needCounted) { int vmmSupported = 0; checkCudaErrors(cuDeviceGetAttribute(&vmmSupported, CU_DEVICE_ATTRIBUTE_VIRTUAL_MEMORY_MANAGEMENT_SUPPORTED, dev)); if (!vmmSupported) { fprintf(stderr, "device does not support virtual memory management\n"); return false; } int unicastSupported = 0; checkCudaErrors(cuDeviceGetAttribute(&unicastSupported, CU_DEVICE_ATTRIBUTE_LOGICAL_ENDPOINT_UNICAST_SUPPORTED, dev)); if (!unicastSupported) { fprintf(stderr, "device does not support unicast logical endpoints\n"); return false; } int fabricSupported = 0; checkCudaErrors(cuDeviceGetAttribute(&fabricSupported, CU_DEVICE_ATTRIBUTE_HANDLE_TYPE_FABRIC_SUPPORTED, dev)); if (!fabricSupported) { fprintf(stderr, "device does not support fabric handles\n"); return false; } // Bitmask of CUlogicalEndpointIpcHandleType. int leHandleTypes = 0; checkCudaErrors(cuDeviceGetAttribute(&leHandleTypes, CU_DEVICE_ATTRIBUTE_LOGICAL_ENDPOINT_SUPPORTED_HANDLE_TYPES, dev)); if ((leHandleTypes & CU_LOGICAL_ENDPOINT_IPC_HANDLE_TYPE_FABRIC) == 0) { fprintf(stderr, "device does not support fabric IPC for logical endpoints\n"); return false; } if (needCounted) { int countedSupported = 0; checkCudaErrors( cuDeviceGetAttribute(&countedSupported, CU_DEVICE_ATTRIBUTE_LOGICAL_ENDPOINT_COUNTED_OPS_SUPPORTED, dev)); if (!countedSupported) { fprintf(stderr, "device does not support counted logical-endpoint ops\n"); return false; } } return true; } static constexpr int DEFAULT_CHUNK_CNT = 100; static void printUsage(const char *prog) { fprintf(stderr, "Usage: mpirun -np %s [chunkCnt] [--counted]\n" " number of MPI ranks (must be >= 2)\n" " [chunkCnt] number of %u-byte chunks each rank sends (default: %d)\n" " --counted use counted fabric puts with a byte-arrival counter\n" "Example: mpirun -np 2 %s\n", prog, FABRIC_CHUNK_SIZE, DEFAULT_CHUNK_CNT, prog); } int main(int argc, char *argv[]) { int chunkCnt = DEFAULT_CHUNK_CNT; bool useCounted = false; for (int i = 1; i < argc; ++i) { if (strcmp(argv[i], "--counted") == 0) { useCounted = true; } else if (strcmp(argv[i], "--help") == 0 || strcmp(argv[i], "-h") == 0) { printUsage(argv[0]); return EXIT_SUCCESS; } else { char *end = nullptr; const long parsed = strtol(argv[i], &end, 10); if (end == argv[i] || *end != '\0' || parsed <= 0 || parsed > INT_MAX) { fprintf(stderr, "error: invalid argument '%s'\n", argv[i]); printUsage(argv[0]); return EXIT_FAILURE; } chunkCnt = static_cast(parsed); } } const size_t totalBytes = static_cast(FABRIC_CHUNK_SIZE) * chunkCnt; // ---- MPI setup: ring topology, rank i sends to (i+1)%N. ---- int myRank = 0; int numRanks = 0; MPI_CHECK(MPI_Init(&argc, &argv)); MPI_CHECK(MPI_Comm_rank(MPI_COMM_WORLD, &myRank)); MPI_CHECK(MPI_Comm_size(MPI_COMM_WORLD, &numRanks)); if (numRanks < 2) { if (myRank == 0) { fprintf(stderr, "error: this example needs at least 2 MPI ranks (got %d)\n", numRanks); } MPI_CHECK(MPI_Finalize()); return EXIT_FAILURE; } if (myRank == 0) { printf("mode = %s, chunkCnt = %d, totalBytes = %zu, ranks = %d\n", useCounted ? "counted" : "put", chunkCnt, totalBytes, numRanks); } // Ring neighbors: put to sendRank; receive completion from recvRank. const int sendRank = (myRank + 1) % numRanks; const int recvRank = (myRank + numRanks - 1) % numRanks; // ---- Device context and stream. ---- // Device ordinals are node-local, so select a device using this process's // rank among the MPI processes on the same node. MPI_Comm localComm = MPI_COMM_NULL; MPI_CHECK(MPI_Comm_split_type(MPI_COMM_WORLD, MPI_COMM_TYPE_SHARED, myRank, MPI_INFO_NULL, &localComm)); int localRank = 0; MPI_CHECK(MPI_Comm_rank(localComm, &localRank)); int deviceCount = 0; checkCudaErrors(cudaGetDeviceCount(&deviceCount)); if (localRank >= deviceCount) { fprintf(stderr, "error: local MPI rank %d cannot be assigned a unique device; " "only %d device(s) are visible on this node\n", localRank, deviceCount); MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE); return EXIT_FAILURE; } const int deviceOrdinal = localRank; checkCudaErrors(cudaSetDevice(deviceOrdinal)); CUdevice cuDevice; checkCudaErrors(cuInit(0)); checkCudaErrors(cuDeviceGet(&cuDevice, deviceOrdinal)); int localSupported = requireDeviceCapabilities(cuDevice, useCounted) ? 1 : 0; int allSupported = 0; MPI_CHECK(MPI_Allreduce(&localSupported, &allSupported, 1, MPI_INT, MPI_MIN, MPI_COMM_WORLD)); if (!allSupported) { MPI_CHECK(MPI_Comm_free(&localComm)); MPI_CHECK(MPI_Finalize()); return EXIT_WAIVED; } CUstream stream; checkCudaErrors(cuStreamCreate(&stream, CU_STREAM_NON_BLOCKING)); // Query how many cliques this device is a member of: size_t cliqueCount = 0; // not modified if there are errors checkCudaErrors(cuDeviceGetCliqueCount(&cliqueCount, cuDevice)); if (cliqueCount == 0) { fprintf(stderr, "This GPU is not a member of any clique."); } // Query the clique info for all cliques std::vector cliqueInfos(cliqueCount); if (cliqueCount > 0) { checkCudaErrors(cuDeviceGetCliqueInfo(cliqueInfos.data(), &cliqueCount, cuDevice)); } unsigned int localCliqueId = 0; bool found = false; for (size_t idx = 0; idx < cliqueCount; ++idx) { if (cliqueInfos[idx].type == CU_CLIQUE_TYPE_UNICAST_LOGICAL_ENDPOINT) { localCliqueId = cliqueInfos[idx].id; found = true; break; } } if (cliqueCount != 0 && !found) { fprintf(stderr, "This GPU is not a member of a clique with unicast logical endpoint support."); } int localCliqueOk = found ? 1 : 0; int allCliqueOk = 0; MPI_CHECK(MPI_Allreduce(&localCliqueOk, &allCliqueOk, 1, MPI_INT, MPI_MIN, MPI_COMM_WORLD)); if (!allCliqueOk) { checkCudaErrors(cuStreamDestroy(stream)); MPI_CHECK(MPI_Comm_free(&localComm)); MPI_CHECK(MPI_Finalize()); return EXIT_WAIVED; } // Verify that all MPI ranks found the same clique id: std::vector allCliqueIds(numRanks); MPI_CHECK(MPI_Allgather(&localCliqueId, sizeof(unsigned int), MPI_BYTE, allCliqueIds.data(), sizeof(unsigned int), MPI_BYTE, MPI_COMM_WORLD)); for (int i = 0; i < numRanks; i++) { if (allCliqueIds[i] != localCliqueId) { fprintf(stderr, "Rank %d: Clique id %u does not match expected %u\n", myRank, allCliqueIds[i], localCliqueId); return EXIT_FAILURE; } } // Reserve enough local ids for this rank's endpoint and all imported peers. CUlogicalEndpointId leId = 0; checkCudaErrors(cuLogicalEndpointIdReserve(&leId, (uint32_t)numRanks)); // ---- Describe the fabric-shareable allocation and query its granularity. ---- CUmemAllocationProp exportAllocProp = {}; exportAllocProp.type = CU_MEM_ALLOCATION_TYPE_PINNED; exportAllocProp.location.type = CU_MEM_LOCATION_TYPE_DEVICE; exportAllocProp.location.id = cuDevice; exportAllocProp.requestedHandleTypes = CU_MEM_HANDLE_TYPE_FABRIC; size_t granularity = 0; checkCudaErrors(cuMemGetAllocationGranularity(&granularity, &exportAllocProp, CU_MEM_ALLOC_GRANULARITY_RECOMMENDED)); // ---- Describe the logical endpoint (unicast; counted ops optional). ---- CUlogicalEndpointProp endpointProp{}; endpointProp.type = CU_LOGICAL_ENDPOINT_TYPE_UNICAST; endpointProp.unicast = {.device = cuDevice}; endpointProp.ipcHandleTypes = CU_LOGICAL_ENDPOINT_IPC_HANDLE_TYPE_FABRIC; endpointProp.flags = useCounted ? CU_LOGICAL_ENDPOINT_FLAG_COUNTED_OPS : CU_LOGICAL_ENDPOINT_FLAG_NONE; cuuint64_t bindAlignment; cuuint64_t maxSize; checkCudaErrors(cuLogicalEndpointGetLimits(&bindAlignment, &maxSize, &endpointProp)); // Make the allocation and endpoint size a multiple of both independent // alignment requirements. const size_t requiredAlignment = std::lcm(granularity, static_cast(bindAlignment)); // ---- Allocate payload, sync region, and alignment padding. ---- size_t exportSize = alignUp(totalBytes + SYNC_REGION_ALIGN_BYTES + SYNC_REGION_BYTES, requiredAlignment); if (exportSize > maxSize) { fprintf(stderr, "error: required endpoint size %zu exceeds maximum %llu\n", exportSize, static_cast(maxSize)); MPI_CHECK(MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE)); return EXIT_FAILURE; } endpointProp.size = exportSize; // ---- Create the local endpoint, then exchange handles with send neighbor. ---- checkCudaErrors(cuLogicalEndpointCreate(leId + myRank, &endpointProp)); // Export our own endpoint to a fabric IPC handle, hand it to the peers that // need it, and import each peer's handle to a locally reserved id. CUlogicalEndpointFabricHandle localHandle; CUlogicalEndpointFabricHandle sendRankHandle; checkCudaErrors(cuLogicalEndpointExport(&localHandle, leId + myRank, CU_LOGICAL_ENDPOINT_IPC_HANDLE_TYPE_FABRIC)); MPI_CHECK(MPI_Sendrecv(&localHandle, sizeof(CUlogicalEndpointFabricHandle), MPI_BYTE, recvRank, 0, &sendRankHandle, sizeof(CUlogicalEndpointFabricHandle), MPI_BYTE, sendRank, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE)); checkCudaErrors(cuLogicalEndpointImport(leId + sendRank, &sendRankHandle, CU_LOGICAL_ENDPOINT_IPC_HANDLE_TYPE_FABRIC)); // Create and import are non-blocking, so wait until both endpoints are ready. waitForEndpointReady(leId + myRank); waitForEndpointReady(leId + sendRank); // ---- Allocate and bind memory after sharing the endpoint. ---- CUmemGenericAllocationHandle exportHandle; CUdeviceptr exportPtr = 0; checkCudaErrors(cuMemCreate(&exportHandle, exportSize, &exportAllocProp, 0)); checkCudaErrors(cuMemAddressReserve(&exportPtr, exportSize, granularity, 0, 0)); checkCudaErrors(cuMemMap(exportPtr, exportSize, 0, exportHandle, 0)); CUmemAccessDesc accessDesc = {}; accessDesc.location.type = CU_MEM_LOCATION_TYPE_DEVICE; accessDesc.location.id = exportAllocProp.location.id; accessDesc.flags = CU_MEM_ACCESS_FLAGS_PROT_READWRITE; checkCudaErrors(cuMemSetAccess(exportPtr, exportSize, &accessDesc, 1)); // Zero the whole allocation before the first transfer or barrier handshake. checkCudaErrors(cuMemsetD8(exportPtr, 0, exportSize)); // 256-byte-aligned sync region after the payload (required for counted fabric puts). std::byte *const basePtr = reinterpret_cast(exportPtr); void *syncPtr = basePtr + totalBytes; size_t syncSpace = exportSize - totalBytes; syncPtr = std::align(SYNC_REGION_ALIGN_BYTES, SYNC_REGION_BYTES, syncPtr, syncSpace); assert(syncPtr != nullptr); // guaranteed by the SYNC_REGION_ALIGN_BYTES slack reserved above const size_t syncRegionOffset = static_cast(syncPtr) - basePtr; // counted mode (--counted): local byte counter the peer's counted puts accumulate into. uint64_t *const counterPtr = reinterpret_cast(syncPtr); // default mode: local inter-GPU flag, set by the receive neighbor's fabric put. uint32_t *const interGpuFlag = reinterpret_cast(syncPtr); // Bind the whole backing allocation at endpoint offset 0, by allocation handle. checkCudaErrors(cuLogicalEndpointBindMem(leId + myRank, cuDevice, 0, exportHandle, 0, exportSize, 0)); // No rank may issue an operation until every destination has been bound. MPI_CHECK(MPI_Barrier(MPI_COMM_WORLD)); // ---- Push data to sendRank. The kernel writes into the neighbor's endpoint // (leId + sendRank); the receive neighbor's puts land in our own buffer. ---- const uint32_t sendLe = leId + sendRank; const uint64_t counterOffset = syncRegionOffset; const uint64_t interGpuFlagOffset = syncRegionOffset; // Launch one block per chunk: gridDim.x == totalBytes / FABRIC_CHUNK_SIZE. if (useCounted) { cftTryPutCountedKernel<<>>(sendLe, counterOffset, myRank, counterPtr, /*expectedBytes=*/totalBytes); } else { // Payload first, then a one-thread flag kernel; same-stream ordering guarantees // the payload puts are done before the inter-GPU flag handshake runs. cftTryPutKernel<<>>(sendLe, myRank); cftSignalFlagKernel<<<1, 1, 0, stream>>>(sendLe, interGpuFlagOffset, interGpuFlag); } checkCudaErrors(cuStreamSynchronize(stream)); // ---- Check the payload the receive neighbor pushed into our endpoint. ---- // Every sending block stages the same rank-tagged pattern, so the expected // value at int index i is recvRank * elemsPerChunk + (i % elemsPerChunk). constexpr size_t elemsPerChunk = FABRIC_CHUNK_SIZE / sizeof(int); const size_t totalElems = totalBytes / sizeof(int); std::vector received(totalElems); checkCudaErrors(cuMemcpyDtoH(received.data(), exportPtr, totalBytes)); size_t mismatches = 0; for (size_t i = 0; i < totalElems; ++i) { const int expected = static_cast(recvRank * elemsPerChunk + i % elemsPerChunk); if (received[i] != expected) { if (mismatches == 0) { fprintf(stderr, "rank %d: first mismatch at int index %zu: got %d, expected %d\n", myRank, i, received[i], expected); } ++mismatches; } } int localFailed = (mismatches != 0) ? 1 : 0; int anyFailed = 0; MPI_CHECK(MPI_Allreduce(&localFailed, &anyFailed, 1, MPI_INT, MPI_MAX, MPI_COMM_WORLD)); if (myRank == 0) { printf("Result = %s\n", anyFailed ? "FAIL" : "PASS"); } // ---- Clean up endpoints and free the backing allocation. ---- // Once every rank's stream has synchronized, all fabric puts // have completed and no rank will issue another operation. Making that global // with a barrier ensures no peer still targets an endpoint we clean up below. MPI_CHECK(MPI_Barrier(MPI_COMM_WORLD)); // Destroy the imported endpoint (the peer target our puts addressed). An import // is only a local reference to the peer's endpoint id and maps no peer memory, // so once our puts have drained this just drops that reference. checkCudaErrors(cuLogicalEndpointDestroy(leId + sendRank)); // Unbind and destroy our owned endpoint, then release the reserved id range. checkCudaErrors(cuLogicalEndpointUnbind(leId + myRank, cuDevice, 0, exportSize)); checkCudaErrors(cuLogicalEndpointDestroy(leId + myRank)); checkCudaErrors(cuLogicalEndpointIdRelease(leId, numRanks)); // Free the backing allocation in VMM cleanup order: unmap the range, release // the physical handle, then free the reserved virtual address range. checkCudaErrors(cuMemUnmap(exportPtr, exportSize)); checkCudaErrors(cuMemRelease(exportHandle)); checkCudaErrors(cuMemAddressFree(exportPtr, exportSize)); checkCudaErrors(cuStreamDestroy(stream)); MPI_CHECK(MPI_Comm_free(&localComm)); MPI_CHECK(MPI_Finalize()); return anyFailed ? EXIT_FAILURE : EXIT_SUCCESS; }