mirror of
https://github.com/NVIDIA/cuda-samples.git
synced 2026-11-18 01:48:16 +08:00
Post v13.5 (#470)
This commit is contained in:
42
cpp/3_CUDA_Features/cftUnicastPut/CMakeLists.txt
Normal file
42
cpp/3_CUDA_Features/cftUnicastPut/CMakeLists.txt
Normal file
@@ -0,0 +1,42 @@
|
||||
cmake_minimum_required(VERSION 3.20)
|
||||
|
||||
# The fabric.try_put instructions and the mbarrier layout::v1 form used by this
|
||||
# sample require __CUDA_ARCH__ >= 1000 (Blackwell and later).
|
||||
set(SAMPLE_DISALLOW_ARCHS 75 80 86 87 89 90)
|
||||
|
||||
include("${CMAKE_CURRENT_SOURCE_DIR}/../../../cmake/DetectCudaArch.cmake")
|
||||
|
||||
if(SAMPLE_SKIP_BUILD)
|
||||
return()
|
||||
endif()
|
||||
|
||||
project(cftUnicastPut LANGUAGES C CXX CUDA)
|
||||
|
||||
include("${CMAKE_CURRENT_SOURCE_DIR}/../../../cmake/CudaSampleCommon.cmake")
|
||||
|
||||
find_package(CUDAToolkit REQUIRED)
|
||||
|
||||
find_package(MPI)
|
||||
|
||||
if(${MPI_FOUND})
|
||||
add_executable(cftUnicastPut cft_unicast_put.cu)
|
||||
|
||||
target_compile_options(cftUnicastPut PRIVATE $<$<COMPILE_LANGUAGE:CUDA>:--extended-lambda>)
|
||||
|
||||
target_compile_features(cftUnicastPut PRIVATE cxx_std_17 cuda_std_17)
|
||||
|
||||
target_include_directories(cftUnicastPut PUBLIC
|
||||
${MPI_INCLUDE_PATH}
|
||||
)
|
||||
|
||||
target_link_libraries(cftUnicastPut PUBLIC
|
||||
${MPI_C_LIBRARIES}
|
||||
${MPI_CXX_LIBRARIES}
|
||||
CUDA::cuda_driver
|
||||
)
|
||||
|
||||
include(InstallSamples)
|
||||
setup_samples_install()
|
||||
else()
|
||||
message(STATUS "MPI not found - will not build sample 'cftUnicastPut'")
|
||||
endif()
|
||||
115
cpp/3_CUDA_Features/cftUnicastPut/README.md
Normal file
115
cpp/3_CUDA_Features/cftUnicastPut/README.md
Normal file
@@ -0,0 +1,115 @@
|
||||
# cftUnicastPut - Unicast Point-to-Point Puts with Compute Fabric Transport
|
||||
|
||||
## Description
|
||||
|
||||
This sample moves data directly between GPUs with **Compute Fabric Transport (CFT)** unicast *logical endpoints*. A logical endpoint (LE) is a fabric-addressable name for a range of device memory: a rank creates an LE, binds VMM fabric memory to it, shares it with peers through a fabric IPC handle, and peers then write into it from device code with `fabric.try_put`, addressing the destination by `(endpoint id, offset)` rather than by a mapped peer pointer.
|
||||
|
||||
The ranks form a ring: rank `r` puts its payload into rank `(r + 1) % N` and expects the payload of rank `(r - 1 + N) % N` to land in its own buffer.
|
||||
|
||||
Two receive-completion schemes are shown, selected on the command line:
|
||||
|
||||
- **Default (plain puts)** — `cftTryPutKernel` only moves payload. A second, one-thread `cftSignalFlagKernel` then puts a 16-byte flag into the neighbor's sync region and spins on its own flag. Because the two kernels are ordered on the same stream, the flag cannot be observed before the payload puts of that rank have completed.
|
||||
- **`--counted` (counted puts)** — `cftTryPutCountedKernel` issues `fabric.try_put_counted`, which additionally increments a byte counter in the destination endpoint. The receiver simply waits until its own counter reaches the number of bytes it expects, so no separate signal kernel is needed.
|
||||
|
||||
After the transfer each rank copies its buffer back to the host and checks the rank-tagged pattern its receive neighbor sent, and rank 0 prints the combined result.
|
||||
|
||||
## What You'll Learn
|
||||
|
||||
How to:
|
||||
- discover the unicast logical-endpoint clique a GPU belongs to and confirm all ranks agree on it
|
||||
- reserve logical-endpoint ids, create an endpoint, and share it across processes with a fabric IPC handle
|
||||
- bind VMM fabric memory to an endpoint and tear the endpoint down safely
|
||||
- issue `fabric.try_put` / `fabric.try_put_counted` from device code and complete them with an `mbarrier`
|
||||
- decode fabric operation errors from the `mbarrier.try_wait` status operand
|
||||
- pick a receive-completion scheme: an explicit inter-GPU signal flag put, or a counted put with a byte-arrival counter
|
||||
|
||||
## Key Concepts
|
||||
|
||||
- **Logical endpoint (LE)** — a fabric-addressable handle for a bound range of device memory; remote writes target `(endpoint id, offset)` instead of a mapped peer address
|
||||
- **Clique** — the set of devices that can address each other's logical endpoints; every participating rank must find the same `CU_CLIQUE_TYPE_UNICAST_LOGICAL_ENDPOINT` clique id
|
||||
- **Fabric put** — an asynchronous device-side write to a peer endpoint issued by a single thread, completed through an `mbarrier` with `complete_tx::16B` accounting
|
||||
- **Counted put** — a fabric put that also accumulates delivered bytes into a 256-byte-aligned counter in the destination endpoint, giving the receiver a completion signal without an extra message
|
||||
- **Fabric error reporting** — when the `mbarrier.try_wait` report predicate is set, `cudaFabricOpErrorStatus*` decodes failures from the `try_wait` status operand.
|
||||
|
||||
## Key APIs
|
||||
|
||||
### CUDA Driver API
|
||||
|
||||
- `cuDeviceGetCliqueCount` / `cuDeviceGetCliqueInfo` — find the unicast logical-endpoint clique of a device
|
||||
- `cuLogicalEndpointIdReserve` / `cuLogicalEndpointIdRelease` — reserve a contiguous range of local endpoint ids
|
||||
- `cuLogicalEndpointCreate` / `cuLogicalEndpointDestroy` — create and destroy an endpoint
|
||||
- `cuLogicalEndpointGetLimits` — query the bind alignment and the maximum endpoint size
|
||||
- `cuLogicalEndpointExport` / `cuLogicalEndpointImport` — share an endpoint through a fabric IPC handle
|
||||
- `cuLogicalEndpointQuery` — wait until a created or imported endpoint is ready to use
|
||||
- `cuLogicalEndpointBindMem` / `cuLogicalEndpointUnbind` — attach and detach the backing allocation
|
||||
- `cuMemCreate`, `cuMemAddressReserve`, `cuMemMap`, `cuMemSetAccess` — allocate the fabric-shareable backing memory
|
||||
|
||||
### Device-side PTX wrappers (`cuda::ptx`)
|
||||
|
||||
- `fabric_try_put` / `fabric_try_put_counted` — issue the remote write, optionally with byte counting
|
||||
- `fabric_submit` — submit the batch of issued fabric operations
|
||||
- `mbarrier_init` (`layout_v1`), `mbarrier_arrive_expect_tx`, `mbarrier_try_wait_parity` — track completion of the puts
|
||||
|
||||
### CUDA Runtime API
|
||||
|
||||
- `cudaFabricOpErrorStatusCount` / `cudaFabricOpErrorStatusGet` — decode fabric errors from the `mbarrier.try_wait` status field (`cudaFabricOpStatusSourceMbarrierV1`)
|
||||
|
||||
## Requirements
|
||||
|
||||
### Hardware
|
||||
|
||||
- Two or more NVIDIA GPUs of compute capability 10.0 or newer.
|
||||
- Two or more NVLink-capable datacenter class NVIDIA GPUs of compute capability 10.0 or newer (`sm_100` / `sm_103` / `sm_107`).
|
||||
- Each rank needs its own device, so a node must expose at least as many GPUs as it runs ranks.
|
||||
|
||||
### Software
|
||||
|
||||
- CUDA Toolkit 13.4 or newer and a driver with `cuLogicalEndpoint*` support
|
||||
- The NVIDIA IMEX daemon with IMEX channels configured, as required for any `CU_MEM_HANDLE_TYPE_FABRIC` sharing
|
||||
- [MPI](../../../README.md#message-passing-interface)
|
||||
- Linux on x86_64 or aarch64
|
||||
|
||||
## How to Build
|
||||
|
||||
See the [top-level README](../../../README.md#building-cuda-samples) for full build instructions, including how to build all samples or a single sample standalone.
|
||||
|
||||
## How to Run
|
||||
|
||||
The sample must be launched under MPI with at least two ranks:
|
||||
|
||||
```bash
|
||||
mpirun -np 2 ./cftUnicastPut [chunkCnt] [--counted]
|
||||
```
|
||||
|
||||
`[chunkCnt]` is optional (default: 1000) and is the number of 8192-byte chunks each rank sends; it also becomes the grid size (one block per chunk). `--counted` switches from plain puts plus a flag handshake to counted puts plus a byte-arrival counter.
|
||||
|
||||
```bash
|
||||
mpirun -np 2 ./cftUnicastPut # defaults: 1000 chunks, plain puts
|
||||
mpirun -np 2 ./cftUnicastPut --counted # defaults: 1000 chunks, counted puts
|
||||
mpirun -np 2 ./cftUnicastPut 1000 # plain puts + inter-GPU flag
|
||||
mpirun -np 2 ./cftUnicastPut 1000 --counted # counted puts + byte counter
|
||||
```
|
||||
|
||||
Debug builds (`-DENABLE_CUDA_DEBUG=ON`, or any build without `NDEBUG`) cap every device-side spin-wait at 30 seconds and print which wait timed out, so a stalled fabric operation reports itself instead of hanging.
|
||||
|
||||
## Expected Output
|
||||
|
||||
```text
|
||||
mode = put, chunkCnt = 1000, totalBytes = 8192000, ranks = 2
|
||||
Result = PASS
|
||||
```
|
||||
|
||||
If any rank's received payload does not match what its neighbor sent, that rank prints the first mismatching element to `stderr`, rank 0 prints `Result = FAIL`, and the sample exits with a non-zero status.
|
||||
|
||||
## Files
|
||||
|
||||
- `cft_unicast_put.cu` — the complete sample: endpoint setup and teardown, both put kernels, the flag-signal kernel, and host-side verification
|
||||
- `README.md` — this file
|
||||
- `CMakeLists.txt` — build configuration
|
||||
|
||||
## See Also
|
||||
|
||||
- [CUDA Compute Fabric Transport](https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/compute-fabric-transport.html)
|
||||
- [CUDA Virtual Memory Management](https://docs.nvidia.com/cuda/cuda-programming-guide/04-special-topics/virtual-memory-management.html)
|
||||
- [CUDA Driver API](https://docs.nvidia.com/cuda/cuda-driver-api/)
|
||||
- [`cuda::ptx` PTX API](https://nvidia.github.io/cccl/unstable/libcudacxx/ptx_api.html)
|
||||
775
cpp/3_CUDA_Features/cftUnicastPut/cft_unicast_put.cu
Normal file
775
cpp/3_CUDA_Features/cftUnicastPut/cft_unicast_put.cu
Normal file
@@ -0,0 +1,775 @@
|
||||
/* Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions
|
||||
* are met:
|
||||
* * Redistributions of source code must retain the above copyright
|
||||
* notice, this list of conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in the
|
||||
* documentation and/or other materials provided with the distribution.
|
||||
* * Neither the name of NVIDIA CORPORATION nor the names of its
|
||||
* contributors may be used to endorse or promote products derived
|
||||
* from this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
|
||||
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
||||
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
|
||||
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
|
||||
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
|
||||
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
|
||||
* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
|
||||
* OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
/*
|
||||
* Unicast point-to-point data movement with Compute Fabric Transport (CFT)
|
||||
* logical endpoints.
|
||||
*
|
||||
* MPI example on a ring: rank r sends to (r+1)%N and receives from (r-1+N)%N.
|
||||
* Default mode uses plain fabric puts; a separate one-thread kernel then flags the
|
||||
* send neighbor over the same endpoint and waits on the receive neighbor's flag.
|
||||
* Pass --counted for counted fabric puts and a counter which counts bytes arrival
|
||||
* for receive completion.
|
||||
*/
|
||||
|
||||
#include <cassert>
|
||||
#include <climits>
|
||||
#include <cstddef>
|
||||
#include <cuda.h>
|
||||
#include <cuda/atomic>
|
||||
#include <cuda/ptx>
|
||||
#include <cuda_awbarrier_helpers.h>
|
||||
#include <cuda_runtime.h>
|
||||
#include <memory>
|
||||
#include <mpi.h>
|
||||
#include <numeric>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <vector>
|
||||
|
||||
#ifndef EXIT_WAIVED
|
||||
#define EXIT_WAIVED 2
|
||||
#endif
|
||||
|
||||
#define MPI_CHECK(cmd) \
|
||||
do { \
|
||||
int e = cmd; \
|
||||
if (e != MPI_SUCCESS) { \
|
||||
printf("Failed: MPI error %s:%d '%d'\n", __FILE__, __LINE__, e); \
|
||||
exit(EXIT_FAILURE); \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
static void checkCuda(CUresult result, const char *func, const char *file, int line)
|
||||
{
|
||||
if (result != CUDA_SUCCESS) {
|
||||
const char *name = nullptr;
|
||||
cuGetErrorName(result, &name);
|
||||
fprintf(stderr,
|
||||
"CUDA driver error at %s:%d code=%d(%s) \"%s\"\n",
|
||||
file,
|
||||
line,
|
||||
static_cast<int>(result),
|
||||
name ? name : "<unknown>",
|
||||
func);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
}
|
||||
|
||||
static void checkCuda(cudaError_t result, const char *func, const char *file, int line)
|
||||
{
|
||||
if (result != cudaSuccess) {
|
||||
fprintf(stderr,
|
||||
"CUDA runtime error at %s:%d code=%d(%s) \"%s\"\n",
|
||||
file,
|
||||
line,
|
||||
static_cast<int>(result),
|
||||
cudaGetErrorName(result),
|
||||
func);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
}
|
||||
|
||||
#define checkCudaErrors(val) checkCuda((val), #val, __FILE__, __LINE__)
|
||||
|
||||
// Round value up to the next multiple of alignment (alignment need not be a power of two).
|
||||
constexpr size_t alignUp(size_t value, size_t alignment)
|
||||
{
|
||||
return (value + alignment - 1) / alignment * alignment;
|
||||
}
|
||||
|
||||
void waitForEndpointReady(CUlogicalEndpointId leId)
|
||||
{
|
||||
while (true) {
|
||||
int ready = 0;
|
||||
checkCudaErrors(cuLogicalEndpointQuery(leId, 1, &ready));
|
||||
if (ready > 0)
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
static constexpr uint32_t FABRIC_CHUNK_SIZE = 8192;
|
||||
// A b128 fabric put moves 16 bytes at a time, and mbarrier complete_tx::16B
|
||||
// counts transferred bytes in 16-byte units. So an expected-tx count is always
|
||||
// the byte count divided by this granularity.
|
||||
static constexpr uint32_t FABRIC_TX_GRANULARITY_BYTES = 16;
|
||||
// Sync region placed after the payload on a 256-byte boundary, reused per mode:
|
||||
// default : a 16-byte inter-GPU flag that the receive neighbor sets with a
|
||||
// fabric put (b128 puts move 16 bytes at a time).
|
||||
// --counted : an 8-byte delivered-byte counter.
|
||||
static constexpr size_t INTER_GPU_FLAG_BYTES = 16;
|
||||
static constexpr size_t SYNC_REGION_BYTES = 16; // max of counter (8B) and flag (16B)
|
||||
// The counter targeted by counted fabric puts must be 256-byte aligned, so the
|
||||
// whole sync region is placed on a 256-byte boundary.
|
||||
static constexpr size_t SYNC_REGION_ALIGN_BYTES = 256;
|
||||
static constexpr int WARP_SIZE = 32;
|
||||
|
||||
#ifndef NDEBUG
|
||||
// Debug builds: cap spin-waits at 30 s so a stuck fabric op prints instead of hanging.
|
||||
static constexpr uint64_t CFT_WAIT_TIMEOUT_NS = 30ULL * 1000 * 1000 * 1000;
|
||||
|
||||
inline __device__ uint64_t readGlobalTimerNs()
|
||||
{
|
||||
uint64_t t = 0;
|
||||
asm volatile("mov.u64 %0, %%globaltimer;" : "=l"(t));
|
||||
return t;
|
||||
}
|
||||
#endif
|
||||
|
||||
// Spin while pred() is true. Returns true when pred() becomes false; in debug
|
||||
// builds returns false after CFT_WAIT_TIMEOUT_NS and prints `what` to identify
|
||||
// the stall. Release builds spin forever. Caller must handle false (e.g. __trap()).
|
||||
template <typename Predicate>
|
||||
inline __device__ bool waitWithDebugTimeout(Predicate &&pred, const char *what)
|
||||
{
|
||||
#ifndef NDEBUG
|
||||
uint64_t start = readGlobalTimerNs();
|
||||
while (pred()) {
|
||||
uint64_t now = readGlobalTimerNs();
|
||||
if (now - start > CFT_WAIT_TIMEOUT_NS) {
|
||||
printf("%s timeout after %llu ns (block=%u)\n", what, (unsigned long long)(now - start), blockIdx.x);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
#else
|
||||
(void)what;
|
||||
while (pred()) {
|
||||
}
|
||||
#endif
|
||||
return true;
|
||||
}
|
||||
|
||||
// Decode and print the fabric errors captured in the mbarrier's report value.
|
||||
// Decode it with cudaFabricOpStatusSourceMbarrierV1: first query the error count, then
|
||||
// retrieve each decoded error. Call this only when the report predicate is set.
|
||||
inline __device__ void reportFabricError(uint8_t *reportValue, uint64_t offset)
|
||||
{
|
||||
const unsigned long long off = offset;
|
||||
unsigned int errCount = 0;
|
||||
if (cudaFabricOpErrorStatusCount(reportValue, cudaFabricOpStatusSourceMbarrierV1, &errCount) != cudaSuccess) {
|
||||
printf("cudaFabricOpErrorStatusCount failed at offset %llu\n", off);
|
||||
return;
|
||||
}
|
||||
for (unsigned int i = 0; i < errCount; ++i) {
|
||||
cudaFabricOpStatusInfo info = cudaFabricOpStatusInfoSuccess;
|
||||
if (cudaFabricOpErrorStatusGet(reportValue, cudaFabricOpStatusSourceMbarrierV1, i, &info) == cudaSuccess) {
|
||||
printf("fabric put error[%u/%u] at offset %llu: status=%d\n", i, errCount, off, (int)info);
|
||||
}
|
||||
else {
|
||||
printf("cudaFabricOpErrorStatusGet failed for error[%u/%u] at offset %llu\n", i, errCount, off);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Each block sends one FABRIC_CHUNK_SIZE slice to the peer's logical endpoint (putLeId).
|
||||
// blockIdx.x selects the chunk: offset = blockIdx.x * FABRIC_CHUNK_SIZE. Launch
|
||||
// gridDim.x = totalBytes / FABRIC_CHUNK_SIZE so every chunk is covered by one block.
|
||||
//
|
||||
// expectedBytes is how many bytes this rank expects to receive (tracked in our
|
||||
// local counter by the peer's puts).
|
||||
__global__ void cftTryPutCountedKernel(uint32_t putLeId,
|
||||
uint64_t counterOffset,
|
||||
int myRank,
|
||||
uint64_t *counterPtr,
|
||||
uint64_t expectedBytes)
|
||||
{
|
||||
constexpr int elems = FABRIC_CHUNK_SIZE / sizeof(int);
|
||||
__shared__ alignas(16) int smemBuf[elems];
|
||||
__shared__ __mbarrier_t putBar;
|
||||
|
||||
if (threadIdx.x == 0) {
|
||||
cuda::ptx::mbarrier_init(cuda::ptx::layout_v1, &putBar, 1);
|
||||
}
|
||||
|
||||
// Fill shared memory with a rank-tagged pattern before streaming it to the peer.
|
||||
for (int i = threadIdx.x; i < elems; i += blockDim.x) {
|
||||
smemBuf[i] = myRank * elems + i;
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
// Thread 0 issues one counted fabric put for this block's chunk.
|
||||
if (threadIdx.x == 0) {
|
||||
const uint64_t offset = blockIdx.x * FABRIC_CHUNK_SIZE;
|
||||
// Make the staged shared-memory source visible to the async proxy before the put reads it.
|
||||
cuda::ptx::fence_proxy_async(cuda::ptx::space_shared);
|
||||
cuda::ptx::fabric_try_put_counted(cuda::ptx::space_shared,
|
||||
cuda::ptx::sem_relaxed,
|
||||
cuda::ptx::scope_sys,
|
||||
putLeId,
|
||||
offset,
|
||||
counterOffset,
|
||||
smemBuf,
|
||||
FABRIC_CHUNK_SIZE,
|
||||
&putBar);
|
||||
cuda::ptx::fabric_submit();
|
||||
cuda::ptx::mbarrier_arrive_expect_tx(cuda::ptx::sem_relaxed,
|
||||
cuda::ptx::scope_cta,
|
||||
cuda::ptx::space_shared,
|
||||
&putBar,
|
||||
(FABRIC_CHUNK_SIZE / FABRIC_TX_GRANULARITY_BYTES));
|
||||
// Spin while the barrier's phase hasn't flipped yet. The report outputs
|
||||
// are only valid once the barrier completes, so inspect them afterward.
|
||||
bool isReportSeen = false;
|
||||
uint8_t reportValue = 0;
|
||||
bool complete = waitWithDebugTimeout(
|
||||
[&]() {
|
||||
return !cuda::ptx::mbarrier_try_wait_parity(cuda::ptx::mbarrier_phase_primary,
|
||||
cuda::ptx::sem_relaxed,
|
||||
cuda::ptx::scope_cta,
|
||||
isReportSeen,
|
||||
reportValue,
|
||||
&putBar,
|
||||
/*phaseParity=*/0);
|
||||
},
|
||||
"cftTryPutCountedKernel: mbarrier try_wait parity");
|
||||
if (!complete) {
|
||||
__trap();
|
||||
}
|
||||
if (isReportSeen) {
|
||||
reportFabricError(&reportValue, offset);
|
||||
}
|
||||
}
|
||||
// Block 0 waits until our receive byte counter reaches expectedBytes. System
|
||||
// scope is required because the peer's counted fabric puts update this
|
||||
// counter in our fabric-bound memory, not via threads in this CTA.
|
||||
// A relaxed load suffices: the fabric-proxy is a strong proxy in the PTX
|
||||
// memory model, so fabric-delivered writes are eventually observable
|
||||
if (blockIdx.x == 0 && threadIdx.x == 0) {
|
||||
cuda::atomic_ref<uint64_t, cuda::thread_scope_system> ref(*counterPtr);
|
||||
if (!waitWithDebugTimeout([&]() { return ref.load(cuda::memory_order_relaxed) != expectedBytes; },
|
||||
"cftTryPutCountedKernel: receive byte counter")) {
|
||||
__trap();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Non-counted variant: each block sends one FABRIC_CHUNK_SIZE chunk via plain
|
||||
// fabric.try_put (no byte counter). This kernel only moves payload; the inter-GPU
|
||||
// completion handshake happens in cftSignalFlagKernel, launched next on the same
|
||||
// stream. The kernel boundary is the cross-block sync point -- once this kernel
|
||||
// returns, every block's put has been issued and its source SMEM consumed.
|
||||
__global__ void cftTryPutKernel(uint32_t putLeId, int myRank)
|
||||
{
|
||||
constexpr int elems = FABRIC_CHUNK_SIZE / sizeof(int);
|
||||
__shared__ alignas(16) int smemBuf[elems];
|
||||
__shared__ __mbarrier_t putBar;
|
||||
|
||||
if (threadIdx.x == 0) {
|
||||
cuda::ptx::mbarrier_init(cuda::ptx::layout_v1, &putBar, 1);
|
||||
}
|
||||
|
||||
for (int i = threadIdx.x; i < elems; i += blockDim.x) {
|
||||
smemBuf[i] = myRank * elems + i;
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
if (threadIdx.x == 0) {
|
||||
const uint64_t offset = blockIdx.x * FABRIC_CHUNK_SIZE;
|
||||
// Make the staged shared-memory source visible to the async proxy before the put reads it.
|
||||
cuda::ptx::fence_proxy_async(cuda::ptx::space_shared);
|
||||
cuda::ptx::fabric_try_put(cuda::ptx::space_shared,
|
||||
cuda::ptx::sem_relaxed,
|
||||
cuda::ptx::scope_sys,
|
||||
putLeId,
|
||||
offset,
|
||||
smemBuf,
|
||||
FABRIC_CHUNK_SIZE,
|
||||
&putBar);
|
||||
cuda::ptx::fabric_submit();
|
||||
cuda::ptx::mbarrier_arrive_expect_tx(cuda::ptx::sem_relaxed,
|
||||
cuda::ptx::scope_cta,
|
||||
cuda::ptx::space_shared,
|
||||
&putBar,
|
||||
(FABRIC_CHUNK_SIZE / FABRIC_TX_GRANULARITY_BYTES));
|
||||
// Spin while the barrier's phase hasn't flipped yet. The report outputs
|
||||
// are only valid once the barrier completes, so inspect them afterward.
|
||||
bool isReportSeen = false;
|
||||
uint8_t reportValue = 0;
|
||||
bool complete = waitWithDebugTimeout(
|
||||
[&]() {
|
||||
return !cuda::ptx::mbarrier_try_wait_parity(cuda::ptx::mbarrier_phase_primary,
|
||||
cuda::ptx::sem_relaxed,
|
||||
cuda::ptx::scope_cta,
|
||||
isReportSeen,
|
||||
reportValue,
|
||||
&putBar,
|
||||
/*phaseParity=*/0);
|
||||
},
|
||||
"cftTryPutKernel: mbarrier try_wait parity");
|
||||
if (!complete) {
|
||||
__trap();
|
||||
}
|
||||
if (isReportSeen) {
|
||||
reportFabricError(&reportValue, offset);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Inter-GPU completion signal for the default (non-counted) path, launched with one
|
||||
// after cftTryPutKernel. Because it is ordered on the same stream, the payload
|
||||
// kernel has already finished, so we issue a single flag put to signal completion.
|
||||
//
|
||||
// Note: this kernel could be overlapped with the tail of cftTryPutKernel using
|
||||
// Programmatic Dependent Launch (PDL) to hide its launch latency. The prologue
|
||||
// (barrier init, staging smemFlag) may run early, but the flag put must still be
|
||||
// gated on payload completion via cudaGridDependencySynchronize(), so we keep the
|
||||
// kernels plainly stream-ordered here for clarity.
|
||||
__global__ void cftSignalFlagKernel(uint32_t putLeId, uint64_t interGpuFlagOffset, uint32_t *interGpuFlag)
|
||||
{
|
||||
__shared__ alignas(16) uint32_t smemFlag[INTER_GPU_FLAG_BYTES / sizeof(uint32_t)];
|
||||
__shared__ __mbarrier_t putBar;
|
||||
|
||||
// This kernel is a single-thread, single-block signal: it is launched <<<1, 1>>>
|
||||
// and its one thread issues the flag put and waits on the neighbor's flag.
|
||||
assert(blockDim.x == 1 && gridDim.x == 1);
|
||||
|
||||
cuda::ptx::mbarrier_init(cuda::ptx::layout_v1, &putBar, 1);
|
||||
// A b128 fabric put moves 16 bytes. Initialize the full source even though
|
||||
// the receiver treats a non-zero first word as the signal.
|
||||
for (size_t i = 0; i < INTER_GPU_FLAG_BYTES / sizeof(uint32_t); ++i) {
|
||||
smemFlag[i] = 1;
|
||||
}
|
||||
cuda::ptx::fence_proxy_async(cuda::ptx::space_shared);
|
||||
cuda::ptx::fabric_try_put(cuda::ptx::space_shared,
|
||||
cuda::ptx::sem_relaxed,
|
||||
cuda::ptx::scope_sys,
|
||||
putLeId,
|
||||
interGpuFlagOffset,
|
||||
smemFlag,
|
||||
INTER_GPU_FLAG_BYTES,
|
||||
&putBar);
|
||||
cuda::ptx::fabric_submit();
|
||||
cuda::ptx::mbarrier_arrive_expect_tx(cuda::ptx::sem_relaxed,
|
||||
cuda::ptx::scope_cta,
|
||||
cuda::ptx::space_shared,
|
||||
&putBar,
|
||||
(INTER_GPU_FLAG_BYTES / FABRIC_TX_GRANULARITY_BYTES));
|
||||
// Spin while the barrier's phase hasn't flipped yet. The report outputs are
|
||||
// only valid once the barrier completes, so inspect them afterward.
|
||||
bool isReportSeen = false;
|
||||
uint8_t reportValue = 0;
|
||||
bool complete = waitWithDebugTimeout(
|
||||
[&]() {
|
||||
return !cuda::ptx::mbarrier_try_wait_parity(cuda::ptx::mbarrier_phase_primary,
|
||||
cuda::ptx::sem_relaxed,
|
||||
cuda::ptx::scope_cta,
|
||||
isReportSeen,
|
||||
reportValue,
|
||||
&putBar,
|
||||
/*phaseParity=*/0);
|
||||
},
|
||||
"cftSignalFlagKernel: mbarrier try_wait parity");
|
||||
if (!complete) {
|
||||
__trap();
|
||||
}
|
||||
if (isReportSeen) {
|
||||
reportFabricError(&reportValue, interGpuFlagOffset);
|
||||
}
|
||||
|
||||
// Wait for the receive neighbor to set our flag via a fabric put.
|
||||
cuda::atomic_ref<uint32_t, cuda::thread_scope_system> ref(*interGpuFlag);
|
||||
if (!waitWithDebugTimeout([&]() { return ref.load(cuda::memory_order_relaxed) == 0; },
|
||||
"cftSignalFlagKernel: inter-GPU flag")) {
|
||||
__trap();
|
||||
}
|
||||
}
|
||||
|
||||
// Confirm the device exposes the capabilities this example relies on: fabric
|
||||
// handles are always required; counted logical-endpoint ops only in --counted mode.
|
||||
// Returns false when the sample is not supported on this device (caller should
|
||||
// exit with EXIT_WAIVED, not EXIT_FAILURE).
|
||||
static bool requireDeviceCapabilities(CUdevice dev, bool needCounted)
|
||||
{
|
||||
int vmmSupported = 0;
|
||||
checkCudaErrors(cuDeviceGetAttribute(&vmmSupported, CU_DEVICE_ATTRIBUTE_VIRTUAL_MEMORY_MANAGEMENT_SUPPORTED, dev));
|
||||
if (!vmmSupported) {
|
||||
fprintf(stderr, "device does not support virtual memory management\n");
|
||||
return false;
|
||||
}
|
||||
|
||||
int unicastSupported = 0;
|
||||
checkCudaErrors(cuDeviceGetAttribute(&unicastSupported, CU_DEVICE_ATTRIBUTE_LOGICAL_ENDPOINT_UNICAST_SUPPORTED, dev));
|
||||
if (!unicastSupported) {
|
||||
fprintf(stderr, "device does not support unicast logical endpoints\n");
|
||||
return false;
|
||||
}
|
||||
|
||||
int fabricSupported = 0;
|
||||
checkCudaErrors(cuDeviceGetAttribute(&fabricSupported, CU_DEVICE_ATTRIBUTE_HANDLE_TYPE_FABRIC_SUPPORTED, dev));
|
||||
if (!fabricSupported) {
|
||||
fprintf(stderr, "device does not support fabric handles\n");
|
||||
return false;
|
||||
}
|
||||
|
||||
// Bitmask of CUlogicalEndpointIpcHandleType.
|
||||
int leHandleTypes = 0;
|
||||
checkCudaErrors(cuDeviceGetAttribute(&leHandleTypes, CU_DEVICE_ATTRIBUTE_LOGICAL_ENDPOINT_SUPPORTED_HANDLE_TYPES, dev));
|
||||
if ((leHandleTypes & CU_LOGICAL_ENDPOINT_IPC_HANDLE_TYPE_FABRIC) == 0) {
|
||||
fprintf(stderr, "device does not support fabric IPC for logical endpoints\n");
|
||||
return false;
|
||||
}
|
||||
|
||||
if (needCounted) {
|
||||
int countedSupported = 0;
|
||||
checkCudaErrors(
|
||||
cuDeviceGetAttribute(&countedSupported, CU_DEVICE_ATTRIBUTE_LOGICAL_ENDPOINT_COUNTED_OPS_SUPPORTED, dev));
|
||||
if (!countedSupported) {
|
||||
fprintf(stderr, "device does not support counted logical-endpoint ops\n");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
static constexpr int DEFAULT_CHUNK_CNT = 100;
|
||||
|
||||
static void printUsage(const char *prog)
|
||||
{
|
||||
fprintf(stderr,
|
||||
"Usage: mpirun -np <N> %s [chunkCnt] [--counted]\n"
|
||||
" <N> number of MPI ranks (must be >= 2)\n"
|
||||
" [chunkCnt] number of %u-byte chunks each rank sends (default: %d)\n"
|
||||
" --counted use counted fabric puts with a byte-arrival counter\n"
|
||||
"Example: mpirun -np 2 %s\n",
|
||||
prog,
|
||||
FABRIC_CHUNK_SIZE,
|
||||
DEFAULT_CHUNK_CNT,
|
||||
prog);
|
||||
}
|
||||
|
||||
int main(int argc, char *argv[])
|
||||
{
|
||||
int chunkCnt = DEFAULT_CHUNK_CNT;
|
||||
bool useCounted = false;
|
||||
for (int i = 1; i < argc; ++i) {
|
||||
if (strcmp(argv[i], "--counted") == 0) {
|
||||
useCounted = true;
|
||||
}
|
||||
else if (strcmp(argv[i], "--help") == 0 || strcmp(argv[i], "-h") == 0) {
|
||||
printUsage(argv[0]);
|
||||
return EXIT_SUCCESS;
|
||||
}
|
||||
else {
|
||||
char *end = nullptr;
|
||||
const long parsed = strtol(argv[i], &end, 10);
|
||||
if (end == argv[i] || *end != '\0' || parsed <= 0 || parsed > INT_MAX) {
|
||||
fprintf(stderr, "error: invalid argument '%s'\n", argv[i]);
|
||||
printUsage(argv[0]);
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
chunkCnt = static_cast<int>(parsed);
|
||||
}
|
||||
}
|
||||
const size_t totalBytes = static_cast<size_t>(FABRIC_CHUNK_SIZE) * chunkCnt;
|
||||
|
||||
// ---- MPI setup: ring topology, rank i sends to (i+1)%N. ----
|
||||
int myRank = 0;
|
||||
int numRanks = 0;
|
||||
MPI_CHECK(MPI_Init(&argc, &argv));
|
||||
MPI_CHECK(MPI_Comm_rank(MPI_COMM_WORLD, &myRank));
|
||||
MPI_CHECK(MPI_Comm_size(MPI_COMM_WORLD, &numRanks));
|
||||
if (numRanks < 2) {
|
||||
if (myRank == 0) {
|
||||
fprintf(stderr, "error: this example needs at least 2 MPI ranks (got %d)\n", numRanks);
|
||||
}
|
||||
MPI_CHECK(MPI_Finalize());
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
if (myRank == 0) {
|
||||
printf("mode = %s, chunkCnt = %d, totalBytes = %zu, ranks = %d\n",
|
||||
useCounted ? "counted" : "put",
|
||||
chunkCnt,
|
||||
totalBytes,
|
||||
numRanks);
|
||||
}
|
||||
// Ring neighbors: put to sendRank; receive completion from recvRank.
|
||||
const int sendRank = (myRank + 1) % numRanks;
|
||||
const int recvRank = (myRank + numRanks - 1) % numRanks;
|
||||
|
||||
// ---- Device context and stream. ----
|
||||
// Device ordinals are node-local, so select a device using this process's
|
||||
// rank among the MPI processes on the same node.
|
||||
MPI_Comm localComm = MPI_COMM_NULL;
|
||||
MPI_CHECK(MPI_Comm_split_type(MPI_COMM_WORLD, MPI_COMM_TYPE_SHARED, myRank, MPI_INFO_NULL, &localComm));
|
||||
int localRank = 0;
|
||||
MPI_CHECK(MPI_Comm_rank(localComm, &localRank));
|
||||
|
||||
int deviceCount = 0;
|
||||
checkCudaErrors(cudaGetDeviceCount(&deviceCount));
|
||||
if (localRank >= deviceCount) {
|
||||
fprintf(stderr,
|
||||
"error: local MPI rank %d cannot be assigned a unique device; "
|
||||
"only %d device(s) are visible on this node\n",
|
||||
localRank,
|
||||
deviceCount);
|
||||
MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE);
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
const int deviceOrdinal = localRank;
|
||||
|
||||
checkCudaErrors(cudaSetDevice(deviceOrdinal));
|
||||
CUdevice cuDevice;
|
||||
checkCudaErrors(cuInit(0));
|
||||
checkCudaErrors(cuDeviceGet(&cuDevice, deviceOrdinal));
|
||||
int localSupported = requireDeviceCapabilities(cuDevice, useCounted) ? 1 : 0;
|
||||
int allSupported = 0;
|
||||
MPI_CHECK(MPI_Allreduce(&localSupported, &allSupported, 1, MPI_INT, MPI_MIN, MPI_COMM_WORLD));
|
||||
if (!allSupported) {
|
||||
MPI_CHECK(MPI_Comm_free(&localComm));
|
||||
MPI_CHECK(MPI_Finalize());
|
||||
return EXIT_WAIVED;
|
||||
}
|
||||
CUstream stream;
|
||||
checkCudaErrors(cuStreamCreate(&stream, CU_STREAM_NON_BLOCKING));
|
||||
|
||||
// Query how many cliques this device is a member of:
|
||||
size_t cliqueCount = 0; // not modified if there are errors
|
||||
checkCudaErrors(cuDeviceGetCliqueCount(&cliqueCount, cuDevice));
|
||||
if (cliqueCount == 0) {
|
||||
fprintf(stderr, "This GPU is not a member of any clique.");
|
||||
}
|
||||
|
||||
// Query the clique info for all cliques
|
||||
std::vector<CUcliqueInfo> cliqueInfos(cliqueCount);
|
||||
if (cliqueCount > 0) {
|
||||
checkCudaErrors(cuDeviceGetCliqueInfo(cliqueInfos.data(), &cliqueCount, cuDevice));
|
||||
}
|
||||
unsigned int localCliqueId = 0;
|
||||
bool found = false;
|
||||
for (size_t idx = 0; idx < cliqueCount; ++idx) {
|
||||
if (cliqueInfos[idx].type == CU_CLIQUE_TYPE_UNICAST_LOGICAL_ENDPOINT) {
|
||||
localCliqueId = cliqueInfos[idx].id;
|
||||
found = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (cliqueCount != 0 && !found) {
|
||||
fprintf(stderr, "This GPU is not a member of a clique with unicast logical endpoint support.");
|
||||
}
|
||||
int localCliqueOk = found ? 1 : 0;
|
||||
int allCliqueOk = 0;
|
||||
MPI_CHECK(MPI_Allreduce(&localCliqueOk, &allCliqueOk, 1, MPI_INT, MPI_MIN, MPI_COMM_WORLD));
|
||||
if (!allCliqueOk) {
|
||||
checkCudaErrors(cuStreamDestroy(stream));
|
||||
MPI_CHECK(MPI_Comm_free(&localComm));
|
||||
MPI_CHECK(MPI_Finalize());
|
||||
return EXIT_WAIVED;
|
||||
}
|
||||
|
||||
// Verify that all MPI ranks found the same clique id:
|
||||
std::vector<unsigned int> allCliqueIds(numRanks);
|
||||
MPI_CHECK(MPI_Allgather(&localCliqueId,
|
||||
sizeof(unsigned int),
|
||||
MPI_BYTE,
|
||||
allCliqueIds.data(),
|
||||
sizeof(unsigned int),
|
||||
MPI_BYTE,
|
||||
MPI_COMM_WORLD));
|
||||
for (int i = 0; i < numRanks; i++) {
|
||||
if (allCliqueIds[i] != localCliqueId) {
|
||||
fprintf(stderr,
|
||||
"Rank %d: Clique id %u does not match expected %u\n",
|
||||
myRank,
|
||||
allCliqueIds[i],
|
||||
localCliqueId);
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
}
|
||||
|
||||
// Reserve enough local ids for this rank's endpoint and all imported peers.
|
||||
CUlogicalEndpointId leId = 0;
|
||||
checkCudaErrors(cuLogicalEndpointIdReserve(&leId, (uint32_t)numRanks));
|
||||
|
||||
// ---- Describe the fabric-shareable allocation and query its granularity. ----
|
||||
CUmemAllocationProp exportAllocProp = {};
|
||||
exportAllocProp.type = CU_MEM_ALLOCATION_TYPE_PINNED;
|
||||
exportAllocProp.location.type = CU_MEM_LOCATION_TYPE_DEVICE;
|
||||
exportAllocProp.location.id = cuDevice;
|
||||
exportAllocProp.requestedHandleTypes = CU_MEM_HANDLE_TYPE_FABRIC;
|
||||
size_t granularity = 0;
|
||||
checkCudaErrors(cuMemGetAllocationGranularity(&granularity, &exportAllocProp, CU_MEM_ALLOC_GRANULARITY_RECOMMENDED));
|
||||
|
||||
// ---- Describe the logical endpoint (unicast; counted ops optional). ----
|
||||
CUlogicalEndpointProp endpointProp{};
|
||||
endpointProp.type = CU_LOGICAL_ENDPOINT_TYPE_UNICAST;
|
||||
endpointProp.unicast = {.device = cuDevice};
|
||||
endpointProp.ipcHandleTypes = CU_LOGICAL_ENDPOINT_IPC_HANDLE_TYPE_FABRIC;
|
||||
endpointProp.flags = useCounted ? CU_LOGICAL_ENDPOINT_FLAG_COUNTED_OPS : CU_LOGICAL_ENDPOINT_FLAG_NONE;
|
||||
cuuint64_t bindAlignment;
|
||||
cuuint64_t maxSize;
|
||||
checkCudaErrors(cuLogicalEndpointGetLimits(&bindAlignment, &maxSize, &endpointProp));
|
||||
// Make the allocation and endpoint size a multiple of both independent
|
||||
// alignment requirements.
|
||||
const size_t requiredAlignment = std::lcm(granularity, static_cast<size_t>(bindAlignment));
|
||||
|
||||
// ---- Allocate payload, sync region, and alignment padding. ----
|
||||
size_t exportSize = alignUp(totalBytes + SYNC_REGION_ALIGN_BYTES + SYNC_REGION_BYTES, requiredAlignment);
|
||||
if (exportSize > maxSize) {
|
||||
fprintf(stderr,
|
||||
"error: required endpoint size %zu exceeds maximum %llu\n",
|
||||
exportSize,
|
||||
static_cast<unsigned long long>(maxSize));
|
||||
MPI_CHECK(MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE));
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
endpointProp.size = exportSize;
|
||||
|
||||
// ---- Create the local endpoint, then exchange handles with send neighbor. ----
|
||||
checkCudaErrors(cuLogicalEndpointCreate(leId + myRank, &endpointProp));
|
||||
// Export our own endpoint to a fabric IPC handle, hand it to the peers that
|
||||
// need it, and import each peer's handle to a locally reserved id.
|
||||
CUlogicalEndpointFabricHandle localHandle;
|
||||
CUlogicalEndpointFabricHandle sendRankHandle;
|
||||
checkCudaErrors(cuLogicalEndpointExport(&localHandle, leId + myRank, CU_LOGICAL_ENDPOINT_IPC_HANDLE_TYPE_FABRIC));
|
||||
MPI_CHECK(MPI_Sendrecv(&localHandle,
|
||||
sizeof(CUlogicalEndpointFabricHandle),
|
||||
MPI_BYTE,
|
||||
recvRank,
|
||||
0,
|
||||
&sendRankHandle,
|
||||
sizeof(CUlogicalEndpointFabricHandle),
|
||||
MPI_BYTE,
|
||||
sendRank,
|
||||
0,
|
||||
MPI_COMM_WORLD,
|
||||
MPI_STATUS_IGNORE));
|
||||
checkCudaErrors(cuLogicalEndpointImport(leId + sendRank, &sendRankHandle, CU_LOGICAL_ENDPOINT_IPC_HANDLE_TYPE_FABRIC));
|
||||
// Create and import are non-blocking, so wait until both endpoints are ready.
|
||||
waitForEndpointReady(leId + myRank);
|
||||
waitForEndpointReady(leId + sendRank);
|
||||
|
||||
// ---- Allocate and bind memory after sharing the endpoint. ----
|
||||
CUmemGenericAllocationHandle exportHandle;
|
||||
CUdeviceptr exportPtr = 0;
|
||||
checkCudaErrors(cuMemCreate(&exportHandle, exportSize, &exportAllocProp, 0));
|
||||
checkCudaErrors(cuMemAddressReserve(&exportPtr, exportSize, granularity, 0, 0));
|
||||
checkCudaErrors(cuMemMap(exportPtr, exportSize, 0, exportHandle, 0));
|
||||
CUmemAccessDesc accessDesc = {};
|
||||
accessDesc.location.type = CU_MEM_LOCATION_TYPE_DEVICE;
|
||||
accessDesc.location.id = exportAllocProp.location.id;
|
||||
accessDesc.flags = CU_MEM_ACCESS_FLAGS_PROT_READWRITE;
|
||||
checkCudaErrors(cuMemSetAccess(exportPtr, exportSize, &accessDesc, 1));
|
||||
// Zero the whole allocation before the first transfer or barrier handshake.
|
||||
checkCudaErrors(cuMemsetD8(exportPtr, 0, exportSize));
|
||||
// 256-byte-aligned sync region after the payload (required for counted fabric puts).
|
||||
std::byte *const basePtr = reinterpret_cast<std::byte *>(exportPtr);
|
||||
void *syncPtr = basePtr + totalBytes;
|
||||
size_t syncSpace = exportSize - totalBytes;
|
||||
syncPtr = std::align(SYNC_REGION_ALIGN_BYTES, SYNC_REGION_BYTES, syncPtr, syncSpace);
|
||||
assert(syncPtr != nullptr); // guaranteed by the SYNC_REGION_ALIGN_BYTES slack reserved above
|
||||
const size_t syncRegionOffset = static_cast<std::byte *>(syncPtr) - basePtr;
|
||||
// counted mode (--counted): local byte counter the peer's counted puts accumulate into.
|
||||
uint64_t *const counterPtr = reinterpret_cast<uint64_t *>(syncPtr);
|
||||
// default mode: local inter-GPU flag, set by the receive neighbor's fabric put.
|
||||
uint32_t *const interGpuFlag = reinterpret_cast<uint32_t *>(syncPtr);
|
||||
// Bind the whole backing allocation at endpoint offset 0, by allocation handle.
|
||||
checkCudaErrors(cuLogicalEndpointBindMem(leId + myRank, cuDevice, 0, exportHandle, 0, exportSize, 0));
|
||||
// No rank may issue an operation until every destination has been bound.
|
||||
MPI_CHECK(MPI_Barrier(MPI_COMM_WORLD));
|
||||
|
||||
// ---- Push data to sendRank. The kernel writes into the neighbor's endpoint
|
||||
// (leId + sendRank); the receive neighbor's puts land in our own buffer. ----
|
||||
const uint32_t sendLe = leId + sendRank;
|
||||
const uint64_t counterOffset = syncRegionOffset;
|
||||
const uint64_t interGpuFlagOffset = syncRegionOffset;
|
||||
|
||||
// Launch one block per chunk: gridDim.x == totalBytes / FABRIC_CHUNK_SIZE.
|
||||
if (useCounted) {
|
||||
cftTryPutCountedKernel<<<chunkCnt, WARP_SIZE, 0, stream>>>(sendLe,
|
||||
counterOffset,
|
||||
myRank,
|
||||
counterPtr,
|
||||
/*expectedBytes=*/totalBytes);
|
||||
}
|
||||
else {
|
||||
// Payload first, then a one-thread flag kernel; same-stream ordering guarantees
|
||||
// the payload puts are done before the inter-GPU flag handshake runs.
|
||||
cftTryPutKernel<<<chunkCnt, WARP_SIZE, 0, stream>>>(sendLe, myRank);
|
||||
cftSignalFlagKernel<<<1, 1, 0, stream>>>(sendLe, interGpuFlagOffset, interGpuFlag);
|
||||
}
|
||||
checkCudaErrors(cuStreamSynchronize(stream));
|
||||
|
||||
// ---- Check the payload the receive neighbor pushed into our endpoint. ----
|
||||
// Every sending block stages the same rank-tagged pattern, so the expected
|
||||
// value at int index i is recvRank * elemsPerChunk + (i % elemsPerChunk).
|
||||
constexpr size_t elemsPerChunk = FABRIC_CHUNK_SIZE / sizeof(int);
|
||||
const size_t totalElems = totalBytes / sizeof(int);
|
||||
std::vector<int> received(totalElems);
|
||||
checkCudaErrors(cuMemcpyDtoH(received.data(), exportPtr, totalBytes));
|
||||
|
||||
size_t mismatches = 0;
|
||||
for (size_t i = 0; i < totalElems; ++i) {
|
||||
const int expected = static_cast<int>(recvRank * elemsPerChunk + i % elemsPerChunk);
|
||||
if (received[i] != expected) {
|
||||
if (mismatches == 0) {
|
||||
fprintf(stderr,
|
||||
"rank %d: first mismatch at int index %zu: got %d, expected %d\n",
|
||||
myRank,
|
||||
i,
|
||||
received[i],
|
||||
expected);
|
||||
}
|
||||
++mismatches;
|
||||
}
|
||||
}
|
||||
|
||||
int localFailed = (mismatches != 0) ? 1 : 0;
|
||||
int anyFailed = 0;
|
||||
MPI_CHECK(MPI_Allreduce(&localFailed, &anyFailed, 1, MPI_INT, MPI_MAX, MPI_COMM_WORLD));
|
||||
if (myRank == 0) {
|
||||
printf("Result = %s\n", anyFailed ? "FAIL" : "PASS");
|
||||
}
|
||||
|
||||
// ---- Clean up endpoints and free the backing allocation. ----
|
||||
// Once every rank's stream has synchronized, all fabric puts
|
||||
// have completed and no rank will issue another operation. Making that global
|
||||
// with a barrier ensures no peer still targets an endpoint we clean up below.
|
||||
MPI_CHECK(MPI_Barrier(MPI_COMM_WORLD));
|
||||
|
||||
// Destroy the imported endpoint (the peer target our puts addressed). An import
|
||||
// is only a local reference to the peer's endpoint id and maps no peer memory,
|
||||
// so once our puts have drained this just drops that reference.
|
||||
checkCudaErrors(cuLogicalEndpointDestroy(leId + sendRank));
|
||||
|
||||
// Unbind and destroy our owned endpoint, then release the reserved id range.
|
||||
checkCudaErrors(cuLogicalEndpointUnbind(leId + myRank, cuDevice, 0, exportSize));
|
||||
checkCudaErrors(cuLogicalEndpointDestroy(leId + myRank));
|
||||
checkCudaErrors(cuLogicalEndpointIdRelease(leId, numRanks));
|
||||
|
||||
// Free the backing allocation in VMM cleanup order: unmap the range, release
|
||||
// the physical handle, then free the reserved virtual address range.
|
||||
checkCudaErrors(cuMemUnmap(exportPtr, exportSize));
|
||||
checkCudaErrors(cuMemRelease(exportHandle));
|
||||
checkCudaErrors(cuMemAddressFree(exportPtr, exportSize));
|
||||
|
||||
checkCudaErrors(cuStreamDestroy(stream));
|
||||
MPI_CHECK(MPI_Comm_free(&localComm));
|
||||
MPI_CHECK(MPI_Finalize());
|
||||
|
||||
return anyFailed ? EXIT_FAILURE : EXIT_SUCCESS;
|
||||
}
|
||||
Reference in New Issue
Block a user