mirror of
https://github.com/NVIDIA/cuda-samples.git
synced 2026-10-23 21:28:17 +08:00
add and update samples with CUDA 11.3 support
This commit is contained in:
Rutwik Choughule
423
Samples/streamOrderedAllocationIPC/Makefile
Normal file
423
Samples/streamOrderedAllocationIPC/Makefile
Normal file
@@ -0,0 +1,423 @@
|
||||
################################################################################
|
||||
# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions
|
||||
# are met:
|
||||
# * Redistributions of source code must retain the above copyright
|
||||
# notice, this list of conditions and the following disclaimer.
|
||||
# * Redistributions in binary form must reproduce the above copyright
|
||||
# notice, this list of conditions and the following disclaimer in the
|
||||
# documentation and/or other materials provided with the distribution.
|
||||
# * Neither the name of NVIDIA CORPORATION nor the names of its
|
||||
# contributors may be used to endorse or promote products derived
|
||||
# from this software without specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
|
||||
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
||||
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
|
||||
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
|
||||
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
|
||||
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
|
||||
# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
|
||||
# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
#
|
||||
################################################################################
|
||||
#
|
||||
# Makefile project only supported on Mac OS X and Linux Platforms)
|
||||
#
|
||||
################################################################################
|
||||
|
||||
# Location of the CUDA Toolkit
|
||||
CUDA_PATH ?= /usr/local/cuda
|
||||
|
||||
##############################
|
||||
# start deprecated interface #
|
||||
##############################
|
||||
ifeq ($(x86_64),1)
|
||||
$(info WARNING - x86_64 variable has been deprecated)
|
||||
$(info WARNING - please use TARGET_ARCH=x86_64 instead)
|
||||
TARGET_ARCH ?= x86_64
|
||||
endif
|
||||
ifeq ($(ARMv7),1)
|
||||
$(info WARNING - ARMv7 variable has been deprecated)
|
||||
$(info WARNING - please use TARGET_ARCH=armv7l instead)
|
||||
TARGET_ARCH ?= armv7l
|
||||
endif
|
||||
ifeq ($(aarch64),1)
|
||||
$(info WARNING - aarch64 variable has been deprecated)
|
||||
$(info WARNING - please use TARGET_ARCH=aarch64 instead)
|
||||
TARGET_ARCH ?= aarch64
|
||||
endif
|
||||
ifeq ($(ppc64le),1)
|
||||
$(info WARNING - ppc64le variable has been deprecated)
|
||||
$(info WARNING - please use TARGET_ARCH=ppc64le instead)
|
||||
TARGET_ARCH ?= ppc64le
|
||||
endif
|
||||
ifneq ($(GCC),)
|
||||
$(info WARNING - GCC variable has been deprecated)
|
||||
$(info WARNING - please use HOST_COMPILER=$(GCC) instead)
|
||||
HOST_COMPILER ?= $(GCC)
|
||||
endif
|
||||
ifneq ($(abi),)
|
||||
$(error ERROR - abi variable has been removed)
|
||||
endif
|
||||
############################
|
||||
# end deprecated interface #
|
||||
############################
|
||||
|
||||
# architecture
|
||||
HOST_ARCH := $(shell uname -m)
|
||||
TARGET_ARCH ?= $(HOST_ARCH)
|
||||
ifneq (,$(filter $(TARGET_ARCH),x86_64 aarch64 sbsa ppc64le armv7l))
|
||||
ifneq ($(TARGET_ARCH),$(HOST_ARCH))
|
||||
ifneq (,$(filter $(TARGET_ARCH),x86_64 aarch64 sbsa ppc64le))
|
||||
TARGET_SIZE := 64
|
||||
else ifneq (,$(filter $(TARGET_ARCH),armv7l))
|
||||
TARGET_SIZE := 32
|
||||
endif
|
||||
else
|
||||
TARGET_SIZE := $(shell getconf LONG_BIT)
|
||||
endif
|
||||
else
|
||||
$(error ERROR - unsupported value $(TARGET_ARCH) for TARGET_ARCH!)
|
||||
endif
|
||||
|
||||
# sbsa and aarch64 systems look similar. Need to differentiate them at host level for now.
|
||||
ifeq ($(HOST_ARCH),aarch64)
|
||||
ifeq ($(CUDA_PATH)/targets/sbsa-linux,$(shell ls -1d $(CUDA_PATH)/targets/sbsa-linux 2>/dev/null))
|
||||
HOST_ARCH := sbsa
|
||||
TARGET_ARCH := sbsa
|
||||
endif
|
||||
endif
|
||||
|
||||
ifneq ($(TARGET_ARCH),$(HOST_ARCH))
|
||||
ifeq (,$(filter $(HOST_ARCH)-$(TARGET_ARCH),aarch64-armv7l x86_64-armv7l x86_64-aarch64 x86_64-sbsa x86_64-ppc64le))
|
||||
$(error ERROR - cross compiling from $(HOST_ARCH) to $(TARGET_ARCH) is not supported!)
|
||||
endif
|
||||
endif
|
||||
|
||||
# When on native aarch64 system with userspace of 32-bit, change TARGET_ARCH to armv7l
|
||||
ifeq ($(HOST_ARCH)-$(TARGET_ARCH)-$(TARGET_SIZE),aarch64-aarch64-32)
|
||||
TARGET_ARCH = armv7l
|
||||
endif
|
||||
|
||||
# operating system
|
||||
HOST_OS := $(shell uname -s 2>/dev/null | tr "[:upper:]" "[:lower:]")
|
||||
TARGET_OS ?= $(HOST_OS)
|
||||
ifeq (,$(filter $(TARGET_OS),linux darwin qnx android))
|
||||
$(error ERROR - unsupported value $(TARGET_OS) for TARGET_OS!)
|
||||
endif
|
||||
|
||||
# host compiler
|
||||
ifeq ($(TARGET_OS),darwin)
|
||||
ifeq ($(shell expr `xcodebuild -version | grep -i xcode | awk '{print $$2}' | cut -d'.' -f1` \>= 5),1)
|
||||
HOST_COMPILER ?= clang++
|
||||
endif
|
||||
else ifneq ($(TARGET_ARCH),$(HOST_ARCH))
|
||||
ifeq ($(HOST_ARCH)-$(TARGET_ARCH),x86_64-armv7l)
|
||||
ifeq ($(TARGET_OS),linux)
|
||||
HOST_COMPILER ?= arm-linux-gnueabihf-g++
|
||||
else ifeq ($(TARGET_OS),qnx)
|
||||
ifeq ($(QNX_HOST),)
|
||||
$(error ERROR - QNX_HOST must be passed to the QNX host toolchain)
|
||||
endif
|
||||
ifeq ($(QNX_TARGET),)
|
||||
$(error ERROR - QNX_TARGET must be passed to the QNX target toolchain)
|
||||
endif
|
||||
export QNX_HOST
|
||||
export QNX_TARGET
|
||||
HOST_COMPILER ?= $(QNX_HOST)/usr/bin/arm-unknown-nto-qnx6.6.0eabi-g++
|
||||
else ifeq ($(TARGET_OS),android)
|
||||
HOST_COMPILER ?= arm-linux-androideabi-g++
|
||||
endif
|
||||
else ifeq ($(TARGET_ARCH),aarch64)
|
||||
ifeq ($(TARGET_OS), linux)
|
||||
HOST_COMPILER ?= aarch64-linux-gnu-g++
|
||||
else ifeq ($(TARGET_OS),qnx)
|
||||
ifeq ($(QNX_HOST),)
|
||||
$(error ERROR - QNX_HOST must be passed to the QNX host toolchain)
|
||||
endif
|
||||
ifeq ($(QNX_TARGET),)
|
||||
$(error ERROR - QNX_TARGET must be passed to the QNX target toolchain)
|
||||
endif
|
||||
export QNX_HOST
|
||||
export QNX_TARGET
|
||||
HOST_COMPILER ?= $(QNX_HOST)/usr/bin/q++
|
||||
else ifeq ($(TARGET_OS), android)
|
||||
HOST_COMPILER ?= aarch64-linux-android-clang++
|
||||
endif
|
||||
else ifeq ($(TARGET_ARCH),sbsa)
|
||||
HOST_COMPILER ?= aarch64-linux-gnu-g++
|
||||
else ifeq ($(TARGET_ARCH),ppc64le)
|
||||
HOST_COMPILER ?= powerpc64le-linux-gnu-g++
|
||||
endif
|
||||
endif
|
||||
HOST_COMPILER ?= g++
|
||||
NVCC := $(CUDA_PATH)/bin/nvcc -ccbin $(HOST_COMPILER)
|
||||
|
||||
# internal flags
|
||||
NVCCFLAGS := -m${TARGET_SIZE}
|
||||
CCFLAGS :=
|
||||
LDFLAGS :=
|
||||
|
||||
# build flags
|
||||
ifeq ($(TARGET_OS),darwin)
|
||||
LDFLAGS += -rpath $(CUDA_PATH)/lib
|
||||
CCFLAGS += -arch $(HOST_ARCH)
|
||||
else ifeq ($(HOST_ARCH)-$(TARGET_ARCH)-$(TARGET_OS),x86_64-armv7l-linux)
|
||||
LDFLAGS += --dynamic-linker=/lib/ld-linux-armhf.so.3
|
||||
CCFLAGS += -mfloat-abi=hard
|
||||
else ifeq ($(TARGET_OS),android)
|
||||
LDFLAGS += -pie
|
||||
CCFLAGS += -fpie -fpic -fexceptions
|
||||
endif
|
||||
|
||||
ifneq ($(TARGET_ARCH),$(HOST_ARCH))
|
||||
ifeq ($(TARGET_ARCH)-$(TARGET_OS),armv7l-linux)
|
||||
ifneq ($(TARGET_FS),)
|
||||
GCCVERSIONLTEQ46 := $(shell expr `$(HOST_COMPILER) -dumpversion` \<= 4.6)
|
||||
ifeq ($(GCCVERSIONLTEQ46),1)
|
||||
CCFLAGS += --sysroot=$(TARGET_FS)
|
||||
endif
|
||||
LDFLAGS += --sysroot=$(TARGET_FS)
|
||||
LDFLAGS += -rpath-link=$(TARGET_FS)/lib
|
||||
LDFLAGS += -rpath-link=$(TARGET_FS)/usr/lib
|
||||
LDFLAGS += -rpath-link=$(TARGET_FS)/usr/lib/arm-linux-gnueabihf
|
||||
endif
|
||||
endif
|
||||
ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-linux)
|
||||
ifneq ($(TARGET_FS),)
|
||||
GCCVERSIONLTEQ46 := $(shell expr `$(HOST_COMPILER) -dumpversion` \<= 4.6)
|
||||
ifeq ($(GCCVERSIONLTEQ46),1)
|
||||
CCFLAGS += --sysroot=$(TARGET_FS)
|
||||
endif
|
||||
LDFLAGS += --sysroot=$(TARGET_FS)
|
||||
LDFLAGS += -rpath-link=$(TARGET_FS)/lib -L$(TARGET_FS)/lib
|
||||
LDFLAGS += -rpath-link=$(TARGET_FS)/lib/aarch64-linux-gnu -L$(TARGET_FS)/lib/aarch64-linux-gnu
|
||||
LDFLAGS += -rpath-link=$(TARGET_FS)/usr/lib -L$(TARGET_FS)/usr/lib
|
||||
LDFLAGS += -rpath-link=$(TARGET_FS)/usr/lib/aarch64-linux-gnu -L$(TARGET_FS)/usr/lib/aarch64-linux-gnu
|
||||
LDFLAGS += --unresolved-symbols=ignore-in-shared-libs
|
||||
CCFLAGS += -isystem=$(TARGET_FS)/usr/include -I$(TARGET_FS)/usr/include -I$(TARGET_FS)/usr/include/libdrm
|
||||
CCFLAGS += -isystem=$(TARGET_FS)/usr/include/aarch64-linux-gnu -I$(TARGET_FS)/usr/include/aarch64-linux-gnu
|
||||
endif
|
||||
endif
|
||||
ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-qnx)
|
||||
NVCCFLAGS += --qpp-config 5.4.0,gcc_ntoaarch64le
|
||||
CCFLAGS += -DWIN_INTERFACE_CUSTOM -I/usr/include/aarch64-qnx-gnu
|
||||
LDFLAGS += -lsocket
|
||||
LDFLAGS += -L/usr/lib/aarch64-qnx-gnu
|
||||
CCFLAGS += "-Wl\,-rpath-link\,/usr/lib/aarch64-qnx-gnu"
|
||||
ifdef TARGET_OVERRIDE
|
||||
LDFLAGS += -lslog2
|
||||
endif
|
||||
|
||||
ifneq ($(TARGET_FS),)
|
||||
LDFLAGS += -L$(TARGET_FS)/usr/lib
|
||||
CCFLAGS += "-Wl\,-rpath-link\,$(TARGET_FS)/usr/lib"
|
||||
LDFLAGS += -L$(TARGET_FS)/usr/libnvidia
|
||||
CCFLAGS += "-Wl\,-rpath-link\,$(TARGET_FS)/usr/libnvidia"
|
||||
CCFLAGS += -I$(TARGET_FS)/../include
|
||||
endif
|
||||
endif
|
||||
endif
|
||||
|
||||
ifdef TARGET_OVERRIDE # cuda toolkit targets override
|
||||
NVCCFLAGS += -target-dir $(TARGET_OVERRIDE)
|
||||
endif
|
||||
|
||||
# Install directory of different arch
|
||||
CUDA_INSTALL_TARGET_DIR :=
|
||||
ifeq ($(TARGET_ARCH)-$(TARGET_OS),armv7l-linux)
|
||||
CUDA_INSTALL_TARGET_DIR = targets/armv7-linux-gnueabihf/
|
||||
else ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-linux)
|
||||
CUDA_INSTALL_TARGET_DIR = targets/aarch64-linux/
|
||||
else ifeq ($(TARGET_ARCH)-$(TARGET_OS),sbsa-linux)
|
||||
CUDA_INSTALL_TARGET_DIR = targets/sbsa-linux/
|
||||
else ifeq ($(TARGET_ARCH)-$(TARGET_OS),armv7l-android)
|
||||
CUDA_INSTALL_TARGET_DIR = targets/armv7-linux-androideabi/
|
||||
else ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-android)
|
||||
CUDA_INSTALL_TARGET_DIR = targets/aarch64-linux-androideabi/
|
||||
else ifeq ($(TARGET_ARCH)-$(TARGET_OS),armv7l-qnx)
|
||||
CUDA_INSTALL_TARGET_DIR = targets/ARMv7-linux-QNX/
|
||||
else ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-qnx)
|
||||
CUDA_INSTALL_TARGET_DIR = targets/aarch64-qnx/
|
||||
else ifeq ($(TARGET_ARCH),ppc64le)
|
||||
CUDA_INSTALL_TARGET_DIR = targets/ppc64le-linux/
|
||||
endif
|
||||
|
||||
# Debug build flags
|
||||
ifeq ($(dbg),1)
|
||||
NVCCFLAGS += -g -G
|
||||
BUILD_TYPE := debug
|
||||
else
|
||||
BUILD_TYPE := release
|
||||
endif
|
||||
|
||||
ALL_CCFLAGS :=
|
||||
ALL_CCFLAGS += $(NVCCFLAGS)
|
||||
ALL_CCFLAGS += $(EXTRA_NVCCFLAGS)
|
||||
ALL_CCFLAGS += $(addprefix -Xcompiler ,$(CCFLAGS))
|
||||
ALL_CCFLAGS += $(addprefix -Xcompiler ,$(EXTRA_CCFLAGS))
|
||||
|
||||
UBUNTU = $(shell lsb_release -i -s 2>/dev/null | grep -i ubuntu)
|
||||
|
||||
SAMPLE_ENABLED := 1
|
||||
|
||||
# This sample is not supported on Mac OSX
|
||||
ifeq ($(TARGET_OS),darwin)
|
||||
$(info >>> WARNING - streamOrderedAllocationIPC is not supported on Mac OSX - waiving sample <<<)
|
||||
SAMPLE_ENABLED := 0
|
||||
endif
|
||||
|
||||
# This sample is not supported on ARMv7
|
||||
ifeq ($(TARGET_ARCH),armv7l)
|
||||
$(info >>> WARNING - streamOrderedAllocationIPC is not supported on ARMv7 - waiving sample <<<)
|
||||
SAMPLE_ENABLED := 0
|
||||
endif
|
||||
|
||||
# This sample is not supported on aarch64
|
||||
ifeq ($(TARGET_ARCH),aarch64)
|
||||
$(info >>> WARNING - streamOrderedAllocationIPC is not supported on aarch64 - waiving sample <<<)
|
||||
SAMPLE_ENABLED := 0
|
||||
endif
|
||||
|
||||
# This sample is not supported on sbsa
|
||||
ifeq ($(TARGET_ARCH),sbsa)
|
||||
$(info >>> WARNING - streamOrderedAllocationIPC is not supported on sbsa - waiving sample <<<)
|
||||
SAMPLE_ENABLED := 0
|
||||
endif
|
||||
|
||||
ALL_LDFLAGS :=
|
||||
ALL_LDFLAGS += $(ALL_CCFLAGS)
|
||||
ALL_LDFLAGS += $(addprefix -Xlinker ,$(LDFLAGS))
|
||||
ALL_LDFLAGS += $(addprefix -Xlinker ,$(EXTRA_LDFLAGS))
|
||||
|
||||
# Common includes and paths for CUDA
|
||||
INCLUDES := -I../../Common
|
||||
LIBRARIES :=
|
||||
|
||||
################################################################################
|
||||
|
||||
# Gencode arguments
|
||||
ifeq ($(TARGET_ARCH),$(filter $(TARGET_ARCH),armv7l aarch64))
|
||||
SMS ?= 35 37 50 52 60 61 70 72 75 80 86
|
||||
else
|
||||
SMS ?= 35 37 50 52 60 61 70 75 80 86
|
||||
endif
|
||||
|
||||
ifeq ($(SMS),)
|
||||
$(info >>> WARNING - no SM architectures have been specified - waiving sample <<<)
|
||||
SAMPLE_ENABLED := 0
|
||||
endif
|
||||
|
||||
ifeq ($(GENCODE_FLAGS),)
|
||||
# Generate SASS code for each SM architecture listed in $(SMS)
|
||||
$(foreach sm,$(SMS),$(eval GENCODE_FLAGS += -gencode arch=compute_$(sm),code=sm_$(sm)))
|
||||
|
||||
# Generate PTX code from the highest SM architecture in $(SMS) to guarantee forward-compatibility
|
||||
HIGHEST_SM := $(lastword $(sort $(SMS)))
|
||||
ifneq ($(HIGHEST_SM),)
|
||||
GENCODE_FLAGS += -gencode arch=compute_$(HIGHEST_SM),code=compute_$(HIGHEST_SM)
|
||||
endif
|
||||
endif
|
||||
|
||||
ifeq ($(TARGET_OS),darwin)
|
||||
ALL_LDFLAGS += -Xcompiler -F/Library/Frameworks -Xlinker -framework -Xlinker CUDA
|
||||
else
|
||||
ifeq ($(TARGET_ARCH),x86_64)
|
||||
CUDA_SEARCH_PATH ?= $(CUDA_PATH)/lib64/stubs
|
||||
CUDA_SEARCH_PATH += $(CUDA_PATH)/targets/x86_64-linux/lib/stubs
|
||||
endif
|
||||
|
||||
ifeq ($(TARGET_ARCH)-$(TARGET_OS),armv7l-linux)
|
||||
CUDA_SEARCH_PATH ?= $(CUDA_PATH)/targets/armv7-linux-gnueabihf/lib/stubs
|
||||
endif
|
||||
|
||||
ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-linux)
|
||||
CUDA_SEARCH_PATH ?= $(CUDA_PATH)/targets/aarch64-linux/lib/stubs
|
||||
endif
|
||||
|
||||
ifeq ($(TARGET_ARCH)-$(TARGET_OS),sbsa-linux)
|
||||
CUDA_SEARCH_PATH ?= $(CUDA_PATH)/targets/sbsa-linux/lib/stubs
|
||||
endif
|
||||
|
||||
ifeq ($(TARGET_ARCH)-$(TARGET_OS),armv7l-android)
|
||||
CUDA_SEARCH_PATH ?= $(CUDA_PATH)/targets/armv7-linux-androideabi/lib/stubs
|
||||
endif
|
||||
|
||||
ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-android)
|
||||
CUDA_SEARCH_PATH ?= $(CUDA_PATH)/targets/aarch64-linux-androideabi/lib/stubs
|
||||
endif
|
||||
|
||||
ifeq ($(TARGET_ARCH)-$(TARGET_OS),armv7l-qnx)
|
||||
CUDA_SEARCH_PATH ?= $(CUDA_PATH)/targets/ARMv7-linux-QNX/lib/stubs
|
||||
endif
|
||||
|
||||
ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-qnx)
|
||||
CUDA_SEARCH_PATH ?= $(CUDA_PATH)/targets/aarch64-qnx/lib/stubs
|
||||
ifdef TARGET_OVERRIDE
|
||||
CUDA_SEARCH_PATH := $(CUDA_PATH)/targets/$(TARGET_OVERRIDE)/lib/stubs
|
||||
endif
|
||||
endif
|
||||
|
||||
ifeq ($(TARGET_ARCH),ppc64le)
|
||||
CUDA_SEARCH_PATH ?= $(CUDA_PATH)/targets/ppc64le-linux/lib/stubs
|
||||
endif
|
||||
|
||||
ifeq ($(HOST_ARCH),ppc64le)
|
||||
CUDA_SEARCH_PATH += $(CUDA_PATH)/lib64/stubs
|
||||
endif
|
||||
|
||||
CUDALIB ?= $(shell find -L $(CUDA_SEARCH_PATH) -maxdepth 1 -name libcuda.so 2> /dev/null)
|
||||
ifeq ("$(CUDALIB)","")
|
||||
$(info >>> WARNING - libcuda.so not found, CUDA Driver is not installed. Please re-install the driver. <<<)
|
||||
SAMPLE_ENABLED := 0
|
||||
else
|
||||
CUDALIB := $(shell echo $(CUDALIB) | sed "s/ .*//" | sed "s/\/libcuda.so//" )
|
||||
LIBRARIES += -L$(CUDALIB) -lcuda
|
||||
endif
|
||||
endif
|
||||
|
||||
ALL_CCFLAGS += --std=c++11 --threads 0
|
||||
|
||||
ifeq ($(SAMPLE_ENABLED),0)
|
||||
EXEC ?= @echo "[@]"
|
||||
endif
|
||||
|
||||
################################################################################
|
||||
|
||||
# Target rules
|
||||
all: build
|
||||
|
||||
build: streamOrderedAllocationIPC
|
||||
|
||||
check.deps:
|
||||
ifeq ($(SAMPLE_ENABLED),0)
|
||||
@echo "Sample will be waived due to the above missing dependencies"
|
||||
else
|
||||
@echo "Sample is ready - all dependencies have been met"
|
||||
endif
|
||||
|
||||
helper_multiprocess.o:../../common/src/helper_multiprocess.cpp
|
||||
$(EXEC) $(NVCC) $(INCLUDES) $(ALL_CCFLAGS) $(GENCODE_FLAGS) -o $@ -c $<
|
||||
|
||||
streamOrderedAllocationIPC.o:streamOrderedAllocationIPC.cu
|
||||
$(EXEC) $(NVCC) $(INCLUDES) $(ALL_CCFLAGS) $(GENCODE_FLAGS) -o $@ -c $<
|
||||
|
||||
streamOrderedAllocationIPC: helper_multiprocess.o streamOrderedAllocationIPC.o
|
||||
$(EXEC) $(NVCC) $(ALL_LDFLAGS) $(GENCODE_FLAGS) -o $@ $+ $(LIBRARIES)
|
||||
$(EXEC) mkdir -p ../../bin/$(TARGET_ARCH)/$(TARGET_OS)/$(BUILD_TYPE)
|
||||
$(EXEC) cp $@ ../../bin/$(TARGET_ARCH)/$(TARGET_OS)/$(BUILD_TYPE)
|
||||
|
||||
run: build
|
||||
$(EXEC) ./streamOrderedAllocationIPC
|
||||
|
||||
clean:
|
||||
rm -f streamOrderedAllocationIPC helper_multiprocess.o streamOrderedAllocationIPC.o
|
||||
rm -rf ../../bin/$(TARGET_ARCH)/$(TARGET_OS)/$(BUILD_TYPE)/streamOrderedAllocationIPC
|
||||
|
||||
clobber: clean
|
||||
65
Samples/streamOrderedAllocationIPC/NsightEclipse.xml
Normal file
65
Samples/streamOrderedAllocationIPC/NsightEclipse.xml
Normal file
@@ -0,0 +1,65 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE entry SYSTEM "SamplesInfo.dtd">
|
||||
<entry>
|
||||
<name>streamOrderedAllocationIPC</name>
|
||||
<cflags>
|
||||
<flag>--std=c++11</flag>
|
||||
</cflags>
|
||||
<cuda_api_list>
|
||||
<toolkit>cudaMallocAsync</toolkit>
|
||||
<toolkit>cudaFreeAsync</toolkit>
|
||||
<toolkit>cudaMemPoolCreate</toolkit>
|
||||
<toolkit>cudaMemPoolImportPointer</toolkit>
|
||||
<toolkit>cudaMemPoolSetAccess</toolkit>
|
||||
<toolkit>cudaMemPoolGetAccess</toolkit>
|
||||
<toolkit>cudaMemPoolExportToShareableHandle</toolkit>
|
||||
<toolkit>cudaMemPoolExportPointer</toolkit>
|
||||
<toolkit>cudaMemPoolDestroy</toolkit>
|
||||
</cuda_api_list>
|
||||
<description><![CDATA[This sample demonstrates IPC pools of stream ordered memory allocated using cudaMallocAsync and cudaMemPool family of APIs.]]></description>
|
||||
<devicecompilation>whole</devicecompilation>
|
||||
<includepaths>
|
||||
<path>./</path>
|
||||
<path>../</path>
|
||||
<path>../../common/inc</path>
|
||||
</includepaths>
|
||||
<keyconcepts>
|
||||
<concept level="basic">Performance Strategies</concept>
|
||||
</keyconcepts>
|
||||
<keywords>
|
||||
</keywords>
|
||||
<libraries>
|
||||
<library os="linux">cuda</library>
|
||||
<library framework="true" os="macosx">CUDA</library>
|
||||
</libraries>
|
||||
<librarypaths>
|
||||
</librarypaths>
|
||||
<nsight_eclipse>true</nsight_eclipse>
|
||||
<primary_file>streamOrderedAllocationIPC.cu</primary_file>
|
||||
<scopes>
|
||||
<scope>1:CUDA Basic Topics</scope>
|
||||
<scope>1:Performance Strategies</scope>
|
||||
</scopes>
|
||||
<sm-arch>sm60</sm-arch>
|
||||
<sm-arch>sm61</sm-arch>
|
||||
<sm-arch>sm70</sm-arch>
|
||||
<sm-arch>sm72</sm-arch>
|
||||
<sm-arch>sm75</sm-arch>
|
||||
<sm-arch>sm80</sm-arch>
|
||||
<sm-arch>sm86</sm-arch>
|
||||
<sources>
|
||||
<extracompilation>../../common/src/helper_multiprocess.cpp</extracompilation>
|
||||
<extraheader>../../common/inc/helper_multiprocess.h</extraheader>
|
||||
</sources>
|
||||
<supported_envs>
|
||||
<env>
|
||||
<arch>x86_64</arch>
|
||||
<platform>linux</platform>
|
||||
</env>
|
||||
</supported_envs>
|
||||
<supported_sm_architectures>
|
||||
<from>6.0</from>
|
||||
</supported_sm_architectures>
|
||||
<title>stream Ordered Allocation IPC Pools</title>
|
||||
<type>exe</type>
|
||||
</entry>
|
||||
60
Samples/streamOrderedAllocationIPC/README.md
Normal file
60
Samples/streamOrderedAllocationIPC/README.md
Normal file
@@ -0,0 +1,60 @@
|
||||
# streamOrderedAllocationIPC - stream Ordered Allocation IPC Pools
|
||||
|
||||
## Description
|
||||
|
||||
This sample demonstrates IPC pools of stream ordered memory allocated using cudaMallocAsync and cudaMemPool family of APIs.
|
||||
|
||||
## Key Concepts
|
||||
|
||||
Performance Strategies
|
||||
|
||||
## Supported SM Architectures
|
||||
|
||||
[SM 6.0 ](https://developer.nvidia.com/cuda-gpus) [SM 6.1 ](https://developer.nvidia.com/cuda-gpus) [SM 7.0 ](https://developer.nvidia.com/cuda-gpus) [SM 7.2 ](https://developer.nvidia.com/cuda-gpus) [SM 7.5 ](https://developer.nvidia.com/cuda-gpus) [SM 8.0 ](https://developer.nvidia.com/cuda-gpus) [SM 8.6 ](https://developer.nvidia.com/cuda-gpus)
|
||||
|
||||
## Supported OSes
|
||||
|
||||
Linux
|
||||
|
||||
## Supported CPU Architecture
|
||||
|
||||
x86_64
|
||||
|
||||
## CUDA APIs involved
|
||||
|
||||
### [CUDA Runtime API](http://docs.nvidia.com/cuda/cuda-runtime-api/index.html)
|
||||
cudaMallocAsync, cudaFreeAsync, cudaMemPoolCreate, cudaMemPoolImportPointer, cudaMemPoolSetAccess, cudaMemPoolGetAccess, cudaMemPoolExportToShareableHandle, cudaMemPoolExportPointer, cudaMemPoolDestroy
|
||||
|
||||
## Prerequisites
|
||||
|
||||
Download and install the [CUDA Toolkit 11.3](https://developer.nvidia.com/cuda-downloads) for your corresponding platform.
|
||||
|
||||
## Build and Run
|
||||
|
||||
### Linux
|
||||
The Linux samples are built using makefiles. To use the makefiles, change the current directory to the sample directory you wish to build, and run make:
|
||||
```
|
||||
$ cd <sample_dir>
|
||||
$ make
|
||||
```
|
||||
The samples makefiles can take advantage of certain options:
|
||||
* **TARGET_ARCH=<arch>** - cross-compile targeting a specific architecture. Allowed architectures are x86_64.
|
||||
By default, TARGET_ARCH is set to HOST_ARCH. On a x86_64 machine, not setting TARGET_ARCH is the equivalent of setting TARGET_ARCH=x86_64.<br/>
|
||||
`$ make TARGET_ARCH=x86_64` <br/>
|
||||
See [here](http://docs.nvidia.com/cuda/cuda-samples/index.html#cross-samples) for more details.
|
||||
* **dbg=1** - build with debug symbols
|
||||
```
|
||||
$ make dbg=1
|
||||
```
|
||||
* **SMS="A B ..."** - override the SM architectures for which the sample will be built, where `"A B ..."` is a space-delimited list of SM architectures. For example, to generate SASS for SM 50 and SM 60, use `SMS="50 60"`.
|
||||
```
|
||||
$ make SMS="50 60"
|
||||
```
|
||||
|
||||
* **HOST_COMPILER=<host_compiler>** - override the default g++ host compiler. See the [Linux Installation Guide](http://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html#system-requirements) for a list of supported host compilers.
|
||||
```
|
||||
$ make HOST_COMPILER=g++
|
||||
```
|
||||
|
||||
## References (for more details)
|
||||
|
||||
440
Samples/streamOrderedAllocationIPC/streamOrderedAllocationIPC.cu
Normal file
440
Samples/streamOrderedAllocationIPC/streamOrderedAllocationIPC.cu
Normal file
@@ -0,0 +1,440 @@
|
||||
/* Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions
|
||||
* are met:
|
||||
* * Redistributions of source code must retain the above copyright
|
||||
* notice, this list of conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in the
|
||||
* documentation and/or other materials provided with the distribution.
|
||||
* * Neither the name of NVIDIA CORPORATION nor the names of its
|
||||
* contributors may be used to endorse or promote products derived
|
||||
* from this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
|
||||
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
||||
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
|
||||
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
|
||||
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
|
||||
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
|
||||
* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
|
||||
* OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
/*
|
||||
* This sample demonstrates Inter Process Communication
|
||||
* using one process per GPU for computation.
|
||||
*/
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <vector>
|
||||
#include <cuda.h>
|
||||
#define CUDA_DRIVER_API 1
|
||||
#include "helper_cuda.h"
|
||||
#include "helper_cuda_drvapi.h"
|
||||
#include "helper_multiprocess.h"
|
||||
|
||||
static const char shmName[] = "streamOrderedAllocationIPCshm";
|
||||
static const char ipcName[] = "streamOrderedAllocationIPC_pipe";
|
||||
// For direct NVLINK and PCI-E peers, at max 8 simultaneous peers are allowed
|
||||
// For NVSWITCH connected peers like DGX-2, simultaneous peers are not limited
|
||||
// in the same way.
|
||||
#define MAX_DEVICES (32)
|
||||
#define DATA_SIZE (64ULL << 20ULL) // 64MB
|
||||
|
||||
#if defined(__linux__)
|
||||
#define cpu_atomic_add32(a, x) __sync_add_and_fetch(a, x)
|
||||
#elif defined(WIN32) || defined(_WIN32) || defined(WIN64) || defined(_WIN64)
|
||||
#define cpu_atomic_add32(a, x) InterlockedAdd((volatile LONG *)a, x)
|
||||
#else
|
||||
#error Unsupported system
|
||||
#endif
|
||||
|
||||
typedef struct shmStruct_st {
|
||||
size_t nprocesses;
|
||||
int barrier;
|
||||
int sense;
|
||||
int devices[MAX_DEVICES];
|
||||
cudaMemPoolPtrExportData exportPtrData[MAX_DEVICES];
|
||||
} shmStruct;
|
||||
|
||||
__global__ void simpleKernel(char *ptr, int sz, char val) {
|
||||
int idx = blockIdx.x * blockDim.x + threadIdx.x;
|
||||
for (; idx < sz; idx += (gridDim.x * blockDim.x)) {
|
||||
ptr[idx] = val;
|
||||
}
|
||||
}
|
||||
|
||||
static void barrierWait(volatile int *barrier, volatile int *sense,
|
||||
unsigned int n) {
|
||||
int count;
|
||||
|
||||
// Check-in
|
||||
count = cpu_atomic_add32(barrier, 1);
|
||||
if (count == n) // Last one in
|
||||
*sense = 1;
|
||||
while (!*sense)
|
||||
;
|
||||
|
||||
// Check-out
|
||||
count = cpu_atomic_add32(barrier, -1);
|
||||
if (count == 0) // Last one out
|
||||
*sense = 0;
|
||||
while (*sense)
|
||||
;
|
||||
}
|
||||
|
||||
static void childProcess(int id) {
|
||||
volatile shmStruct *shm = NULL;
|
||||
cudaStream_t stream;
|
||||
sharedMemoryInfo info;
|
||||
size_t procCount, i;
|
||||
int blocks = 0;
|
||||
int threads = 128;
|
||||
cudaDeviceProp prop;
|
||||
std::vector<void *> ptrs;
|
||||
|
||||
std::vector<char> verification_buffer(DATA_SIZE);
|
||||
|
||||
ipcHandle *ipcChildHandle = NULL;
|
||||
checkIpcErrors(ipcOpenSocket(ipcChildHandle));
|
||||
|
||||
if (sharedMemoryOpen(shmName, sizeof(shmStruct), &info) != 0) {
|
||||
printf("Failed to create shared memory slab\n");
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
shm = (volatile shmStruct *)info.addr;
|
||||
procCount = shm->nprocesses;
|
||||
|
||||
barrierWait(&shm->barrier, &shm->sense, (unsigned int)(procCount + 1));
|
||||
|
||||
// Receive all allocation handles shared by Parent.
|
||||
std::vector<ShareableHandle> shHandle(shm->nprocesses);
|
||||
checkIpcErrors(ipcRecvShareableHandles(ipcChildHandle, shHandle));
|
||||
|
||||
checkCudaErrors(cudaSetDevice(shm->devices[id]));
|
||||
checkCudaErrors(cudaGetDeviceProperties(&prop, shm->devices[id]));
|
||||
checkCudaErrors(cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking));
|
||||
checkCudaErrors(cudaOccupancyMaxActiveBlocksPerMultiprocessor(
|
||||
&blocks, simpleKernel, threads, 0));
|
||||
blocks *= prop.multiProcessorCount;
|
||||
|
||||
std::vector<cudaMemPool_t> pools(shm->nprocesses);
|
||||
|
||||
cudaMemAllocationHandleType handleType = cudaMemHandleTypePosixFileDescriptor;
|
||||
|
||||
// Import mem pools from all the devices created in the master
|
||||
// process using shareable handles received via socket
|
||||
// and import the pointer to the allocated buffer using
|
||||
// exportData filled in shared memory by the master process.
|
||||
for (i = 0; i < procCount; i++) {
|
||||
checkCudaErrors(cudaMemPoolImportFromShareableHandle(
|
||||
&pools[i], (void *)shHandle[i], handleType, 0));
|
||||
|
||||
cudaMemAccessFlags accessFlags;
|
||||
cudaMemLocation location;
|
||||
location.type = cudaMemLocationTypeDevice;
|
||||
location.id = shm->devices[id];
|
||||
checkCudaErrors(cudaMemPoolGetAccess(&accessFlags, pools[i], &location));
|
||||
if (accessFlags != cudaMemAccessFlagsProtReadWrite) {
|
||||
cudaMemAccessDesc desc;
|
||||
memset(&desc, 0, sizeof(cudaMemAccessDesc));
|
||||
desc.location.type = cudaMemLocationTypeDevice;
|
||||
desc.location.id = shm->devices[id];
|
||||
desc.flags = cudaMemAccessFlagsProtReadWrite;
|
||||
checkCudaErrors(cudaMemPoolSetAccess(pools[i], &desc, 1));
|
||||
}
|
||||
|
||||
// Import the allocation from each memory pool by iterating over exportData
|
||||
// until import is success.
|
||||
for (int j = 0; j < procCount; j++) {
|
||||
void *ptr = NULL;
|
||||
// Import the allocation using the opaque export data retrieved through
|
||||
// the shared memory".
|
||||
cudaError_t ret = cudaMemPoolImportPointer(
|
||||
&ptr, pools[i], (cudaMemPoolPtrExportData *)&shm->exportPtrData[j]);
|
||||
|
||||
if (ret == cudaSuccess) {
|
||||
// Pointer import is successful hence add it to the ptrs bag.
|
||||
ptrs.push_back(ptr);
|
||||
break;
|
||||
} else {
|
||||
// Reset failure error received from cudaMemPoolImportPointer
|
||||
// for further try.
|
||||
cudaGetLastError();
|
||||
}
|
||||
}
|
||||
// Since we have imported allocations shared by the parent with us, we can
|
||||
// close this ShareableHandle.
|
||||
checkIpcErrors(ipcCloseShareableHandle(shHandle[i]));
|
||||
}
|
||||
|
||||
// Since we have imported allocations shared by the parent with us, we can
|
||||
// close the socket.
|
||||
checkIpcErrors(ipcCloseSocket(ipcChildHandle));
|
||||
|
||||
// At each iteration of the loop, each sibling process will push work on
|
||||
// their respective devices accessing the next peer mapped buffer allocated
|
||||
// by the master process (these can come from other sibling processes as
|
||||
// well). To coordinate each process' access, we force the stream to wait for
|
||||
// the work already accessing this buffer.
|
||||
for (i = 0; i < procCount; i++) {
|
||||
size_t bufferId = (i + id) % procCount;
|
||||
|
||||
// Push a simple kernel on it
|
||||
simpleKernel<<<blocks, threads, 0, stream>>>((char *)ptrs[bufferId],
|
||||
DATA_SIZE, id);
|
||||
checkCudaErrors(cudaGetLastError());
|
||||
checkCudaErrors(cudaStreamSynchronize(stream));
|
||||
|
||||
// Wait for all my sibling processes to push this stage of their work
|
||||
// before proceeding to the next. This prevents siblings from racing
|
||||
// ahead and clobbering the recorded event or waiting on the wrong
|
||||
// recorded event.
|
||||
barrierWait(&shm->barrier, &shm->sense, (unsigned int)procCount);
|
||||
if (id == 0) {
|
||||
printf("Step %lld done\n", (unsigned long long)i);
|
||||
}
|
||||
}
|
||||
|
||||
// Now wait for my buffer to be ready so I can copy it locally and verify it
|
||||
checkCudaErrors(cudaMemcpyAsync(&verification_buffer[0], ptrs[id], DATA_SIZE,
|
||||
cudaMemcpyDeviceToHost, stream));
|
||||
|
||||
// And wait for all the queued up work to complete
|
||||
checkCudaErrors(cudaStreamSynchronize(stream));
|
||||
|
||||
printf("Process %d: verifying...\n", id);
|
||||
|
||||
// The contents should have the id of the sibling just after me
|
||||
char compareId = (char)((id + 1) % procCount);
|
||||
for (unsigned long long j = 0; j < DATA_SIZE; j++) {
|
||||
if (verification_buffer[j] != compareId) {
|
||||
printf("Process %d: Verification mismatch at %lld: %d != %d\n", id, j,
|
||||
(int)verification_buffer[j], (int)compareId);
|
||||
}
|
||||
}
|
||||
|
||||
// Clean up!
|
||||
for (i = 0; i < procCount; i++) {
|
||||
// Free the memory before the exporter process frees it
|
||||
checkCudaErrors(cudaFreeAsync(ptrs[i], stream));
|
||||
}
|
||||
|
||||
// And wait for all the queued up work to complete
|
||||
checkCudaErrors(cudaStreamSynchronize(stream));
|
||||
checkCudaErrors(cudaStreamDestroy(stream));
|
||||
|
||||
printf("Process %d complete!\n", id);
|
||||
}
|
||||
|
||||
static void parentProcess(char *app) {
|
||||
sharedMemoryInfo info;
|
||||
int devCount, i;
|
||||
volatile shmStruct *shm = NULL;
|
||||
std::vector<void *> ptrs;
|
||||
std::vector<Process> processes;
|
||||
|
||||
checkCudaErrors(cudaGetDeviceCount(&devCount));
|
||||
std::vector<CUdevice> devices(devCount);
|
||||
for (i = 0; i < devCount; i++) {
|
||||
cuDeviceGet(&devices[i], i);
|
||||
}
|
||||
|
||||
if (sharedMemoryCreate(shmName, sizeof(*shm), &info) != 0) {
|
||||
printf("Failed to create shared memory slab\n");
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
shm = (volatile shmStruct *)info.addr;
|
||||
memset((void *)shm, 0, sizeof(*shm));
|
||||
|
||||
// Pick all the devices that can access each other's memory for this test
|
||||
// Keep in mind that CUDA has minimal support for fork() without a
|
||||
// corresponding exec() in the child process, but in this case our
|
||||
// spawnProcess will always exec, so no need to worry.
|
||||
for (i = 0; i < devCount; i++) {
|
||||
bool allPeers = true;
|
||||
cudaDeviceProp prop;
|
||||
checkCudaErrors(cudaGetDeviceProperties(&prop, i));
|
||||
|
||||
int isMemPoolSupported = 0;
|
||||
checkCudaErrors(cudaDeviceGetAttribute(&isMemPoolSupported,
|
||||
cudaDevAttrMemoryPoolsSupported, i));
|
||||
// CUDA IPC is only supported on devices with unified addressing
|
||||
if (!isMemPoolSupported) {
|
||||
printf("Device %d does not support cuda memory pools, skipping...\n", i);
|
||||
continue;
|
||||
}
|
||||
int deviceSupportsIpcHandle = 0;
|
||||
#if defined(__linux__)
|
||||
checkCudaErrors(cuDeviceGetAttribute(
|
||||
&deviceSupportsIpcHandle,
|
||||
CU_DEVICE_ATTRIBUTE_HANDLE_TYPE_POSIX_FILE_DESCRIPTOR_SUPPORTED,
|
||||
devices[i]));
|
||||
#else
|
||||
cuDeviceGetAttribute(&deviceSupportsIpcHandle,
|
||||
CU_DEVICE_ATTRIBUTE_HANDLE_TYPE_WIN32_HANDLE_SUPPORTED,
|
||||
devices[i]);
|
||||
#endif
|
||||
|
||||
if (!deviceSupportsIpcHandle) {
|
||||
printf("Device %d does not support CUDA IPC Handle, skipping...\n", i);
|
||||
continue;
|
||||
}
|
||||
// This sample requires two processes accessing each device, so we need
|
||||
// to ensure exclusive or prohibited mode is not set
|
||||
if (prop.computeMode != cudaComputeModeDefault) {
|
||||
printf("Device %d is in an unsupported compute mode for this sample\n",
|
||||
i);
|
||||
continue;
|
||||
}
|
||||
#if defined(WIN32) || defined(_WIN32) || defined(WIN64) || defined(_WIN64)
|
||||
// CUDA IPC on Windows is only supported on TCC
|
||||
if (!prop.tccDriver) {
|
||||
printf("Device %d is not in TCC mode\n", i);
|
||||
continue;
|
||||
}
|
||||
#endif
|
||||
|
||||
for (int j = 0; j < shm->nprocesses; j++) {
|
||||
int canAccessPeerIJ, canAccessPeerJI;
|
||||
checkCudaErrors(
|
||||
cudaDeviceCanAccessPeer(&canAccessPeerJI, shm->devices[j], i));
|
||||
checkCudaErrors(
|
||||
cudaDeviceCanAccessPeer(&canAccessPeerIJ, i, shm->devices[j]));
|
||||
if (!canAccessPeerIJ || !canAccessPeerJI) {
|
||||
allPeers = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (allPeers) {
|
||||
// Enable peers here. This isn't necessary for IPC, but it will
|
||||
// setup the peers for the device. For systems that only allow 8
|
||||
// peers per GPU at a time, this acts to remove devices from CanAccessPeer
|
||||
for (int j = 0; j < shm->nprocesses; j++) {
|
||||
checkCudaErrors(cudaSetDevice(i));
|
||||
checkCudaErrors(cudaDeviceEnablePeerAccess(shm->devices[j], 0));
|
||||
checkCudaErrors(cudaSetDevice(shm->devices[j]));
|
||||
checkCudaErrors(cudaDeviceEnablePeerAccess(i, 0));
|
||||
}
|
||||
shm->devices[shm->nprocesses++] = i;
|
||||
if (shm->nprocesses >= MAX_DEVICES) break;
|
||||
} else {
|
||||
printf(
|
||||
"Device %d is not peer capable with some other selected peers, "
|
||||
"skipping\n",
|
||||
i);
|
||||
}
|
||||
}
|
||||
|
||||
if (shm->nprocesses == 0) {
|
||||
printf("No CUDA devices support IPC\n");
|
||||
exit(EXIT_WAIVED);
|
||||
}
|
||||
|
||||
std::vector<ShareableHandle> shareableHandles(shm->nprocesses);
|
||||
std::vector<cudaStream_t> streams(shm->nprocesses);
|
||||
std::vector<cudaMemPool_t> pools(shm->nprocesses);
|
||||
|
||||
// Now allocate memory for each process and fill the shared
|
||||
// memory buffer with the export data and get memPool handles to communicate
|
||||
for (i = 0; i < shm->nprocesses; i++) {
|
||||
void *ptr = NULL;
|
||||
checkCudaErrors(cudaSetDevice(shm->devices[i]));
|
||||
checkCudaErrors(
|
||||
cudaStreamCreateWithFlags(&streams[i], cudaStreamNonBlocking));
|
||||
// Allocate an explicit pool with IPC capabilities
|
||||
cudaMemPoolProps poolProps;
|
||||
memset(&poolProps, 0, sizeof(cudaMemPoolProps));
|
||||
poolProps.allocType = cudaMemAllocationTypePinned;
|
||||
poolProps.handleTypes = cudaMemHandleTypePosixFileDescriptor;
|
||||
|
||||
poolProps.location.type = cudaMemLocationTypeDevice;
|
||||
poolProps.location.id = shm->devices[i];
|
||||
|
||||
checkCudaErrors(cudaMemPoolCreate(&pools[i], &poolProps));
|
||||
|
||||
// Query the shareable handle for the pool
|
||||
cudaMemAllocationHandleType handleType =
|
||||
cudaMemHandleTypePosixFileDescriptor;
|
||||
// Allocate memory in a stream from the pool just created
|
||||
checkCudaErrors(cudaMallocAsync(&ptr, DATA_SIZE, pools[i], streams[i]));
|
||||
|
||||
checkCudaErrors(cudaMemPoolExportToShareableHandle(
|
||||
&shareableHandles[i], pools[i], handleType, 0));
|
||||
|
||||
// Get the opaque ‘bag-of-bits’ representing the allocation
|
||||
memset((void *)&shm->exportPtrData[i], 0, sizeof(cudaMemPoolPtrExportData));
|
||||
checkCudaErrors(cudaMemPoolExportPointer(
|
||||
(cudaMemPoolPtrExportData *)&shm->exportPtrData[i], ptr));
|
||||
ptrs.push_back(ptr);
|
||||
}
|
||||
|
||||
// Launch the child processes!
|
||||
for (i = 0; i < shm->nprocesses; i++) {
|
||||
char devIdx[10];
|
||||
char *const args[] = {app, devIdx, NULL};
|
||||
Process process;
|
||||
|
||||
SPRINTF(devIdx, "%d", i);
|
||||
|
||||
if (spawnProcess(&process, app, args)) {
|
||||
printf("Failed to create process\n");
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
processes.push_back(process);
|
||||
}
|
||||
|
||||
barrierWait(&shm->barrier, &shm->sense, (unsigned int)(shm->nprocesses + 1));
|
||||
|
||||
ipcHandle *ipcParentHandle = NULL;
|
||||
checkIpcErrors(ipcCreateSocket(ipcParentHandle, ipcName, processes));
|
||||
checkIpcErrors(
|
||||
ipcSendShareableHandles(ipcParentHandle, shareableHandles, processes));
|
||||
|
||||
// Close the shareable handles as they are not needed anymore.
|
||||
for (int i = 0; i < shm->nprocesses; i++) {
|
||||
checkIpcErrors(ipcCloseShareableHandle(shareableHandles[i]));
|
||||
}
|
||||
checkIpcErrors(ipcCloseSocket(ipcParentHandle));
|
||||
|
||||
// And wait for them to finish
|
||||
for (i = 0; i < processes.size(); i++) {
|
||||
if (waitProcess(&processes[i]) != EXIT_SUCCESS) {
|
||||
printf("Process %d failed!\n", i);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
}
|
||||
|
||||
// Clean up!
|
||||
for (i = 0; i < shm->nprocesses; i++) {
|
||||
checkCudaErrors(cudaSetDevice(shm->devices[i]));
|
||||
checkCudaErrors(cudaFreeAsync(ptrs[i], streams[i]));
|
||||
checkCudaErrors(cudaStreamSynchronize(streams[i]));
|
||||
checkCudaErrors(cudaMemPoolDestroy(pools[i]));
|
||||
}
|
||||
|
||||
sharedMemoryClose(&info);
|
||||
}
|
||||
|
||||
// Host code
|
||||
int main(int argc, char **argv) {
|
||||
#if defined(__arm__) || defined(__aarch64__) || defined(WIN32) || \
|
||||
defined(_WIN32) || defined(WIN64) || defined(_WIN64)
|
||||
printf("Not supported on ARM\n");
|
||||
return EXIT_WAIVED;
|
||||
#else
|
||||
if (argc == 1) {
|
||||
parentProcess(argv[0]);
|
||||
} else {
|
||||
childProcess(atoi(argv[1]));
|
||||
}
|
||||
return EXIT_SUCCESS;
|
||||
#endif
|
||||
}
|
||||
Reference in New Issue
Block a user