add and update samples for CUDA 11.6

Samples/5_Domain_Specific/nbody/.vscode/c_cpp_properties.json

@@ -0,0 +1,18 @@
+{
+    "configurations": [
+        {
+            "name": "Linux",
+            "includePath": [
+                "${workspaceFolder}/**",
+                "${workspaceFolder}/../../../Common"
+            ],
+            "defines": [],
+            "compilerPath": "/usr/local/cuda/bin/nvcc",
+            "cStandard": "gnu17",
+            "cppStandard": "gnu++14",
+            "intelliSenseMode": "linux-gcc-x64",
+            "configurationProvider": "ms-vscode.makefile-tools"
+        }
+    ],
+    "version": 4
+}

Samples/5_Domain_Specific/nbody/.vscode/extensions.json

@@ -0,0 +1,7 @@
+{
+    "recommendations": [
+        "nvidia.nsight-vscode-edition",
+        "ms-vscode.cpptools",
+        "ms-vscode.makefile-tools"
+    ]
+}

Samples/5_Domain_Specific/nbody/.vscode/launch.json

@@ -0,0 +1,10 @@
+{
+    "configurations": [
+        {
+            "name": "CUDA C++: Launch",
+            "type": "cuda-gdb",
+            "request": "launch",
+            "program": "${workspaceFolder}/nbody"
+        }
+    ]
+}

Samples/5_Domain_Specific/nbody/.vscode/tasks.json

@@ -0,0 +1,15 @@
+{
+    "version": "2.0.0",
+    "tasks": [
+        {
+            "label": "sample",
+            "type": "shell",
+            "command": "make dbg=1",
+            "problemMatcher": ["$nvcc"],
+            "group": {
+                "kind": "build",
+                "isDefault": true
+            }
+        }
+    ]
+}

Samples/5_Domain_Specific/nbody/Makefile

@@ -0,0 +1,367 @@
+################################################################################
+# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#  * Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#  * Neither the name of NVIDIA CORPORATION nor the names of its
+#    contributors may be used to endorse or promote products derived
+#    from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+################################################################################
+#
+# Makefile project only supported on Mac OS X and Linux Platforms)
+#
+################################################################################
+
+# Location of the CUDA Toolkit
+CUDA_PATH ?= /usr/local/cuda
+
+##############################
+# start deprecated interface #
+##############################
+ifeq ($(x86_64),1)
+    $(info WARNING - x86_64 variable has been deprecated)
+    $(info WARNING - please use TARGET_ARCH=x86_64 instead)
+    TARGET_ARCH ?= x86_64
+endif
+ifeq ($(ARMv7),1)
+    $(info WARNING - ARMv7 variable has been deprecated)
+    $(info WARNING - please use TARGET_ARCH=armv7l instead)
+    TARGET_ARCH ?= armv7l
+endif
+ifeq ($(aarch64),1)
+    $(info WARNING - aarch64 variable has been deprecated)
+    $(info WARNING - please use TARGET_ARCH=aarch64 instead)
+    TARGET_ARCH ?= aarch64
+endif
+ifeq ($(ppc64le),1)
+    $(info WARNING - ppc64le variable has been deprecated)
+    $(info WARNING - please use TARGET_ARCH=ppc64le instead)
+    TARGET_ARCH ?= ppc64le
+endif
+ifneq ($(GCC),)
+    $(info WARNING - GCC variable has been deprecated)
+    $(info WARNING - please use HOST_COMPILER=$(GCC) instead)
+    HOST_COMPILER ?= $(GCC)
+endif
+ifneq ($(abi),)
+    $(error ERROR - abi variable has been removed)
+endif
+############################
+# end deprecated interface #
+############################
+
+# architecture
+HOST_ARCH   := $(shell uname -m)
+TARGET_ARCH ?= $(HOST_ARCH)
+ifneq (,$(filter $(TARGET_ARCH),x86_64 aarch64 sbsa ppc64le armv7l))
+    ifneq ($(TARGET_ARCH),$(HOST_ARCH))
+        ifneq (,$(filter $(TARGET_ARCH),x86_64 aarch64 sbsa ppc64le))
+            TARGET_SIZE := 64
+        else ifneq (,$(filter $(TARGET_ARCH),armv7l))
+            TARGET_SIZE := 32
+        endif
+    else
+        TARGET_SIZE := $(shell getconf LONG_BIT)
+    endif
+else
+    $(error ERROR - unsupported value $(TARGET_ARCH) for TARGET_ARCH!)
+endif
+
+# sbsa and aarch64 systems look similar. Need to differentiate them at host level for now.
+ifeq ($(HOST_ARCH),aarch64)
+    ifeq ($(CUDA_PATH)/targets/sbsa-linux,$(shell ls -1d $(CUDA_PATH)/targets/sbsa-linux 2>/dev/null))
+        HOST_ARCH := sbsa
+        TARGET_ARCH := sbsa
+    endif
+endif
+
+ifneq ($(TARGET_ARCH),$(HOST_ARCH))
+    ifeq (,$(filter $(HOST_ARCH)-$(TARGET_ARCH),aarch64-armv7l x86_64-armv7l x86_64-aarch64 x86_64-sbsa x86_64-ppc64le))
+        $(error ERROR - cross compiling from $(HOST_ARCH) to $(TARGET_ARCH) is not supported!)
+    endif
+endif
+
+# When on native aarch64 system with userspace of 32-bit, change TARGET_ARCH to armv7l
+ifeq ($(HOST_ARCH)-$(TARGET_ARCH)-$(TARGET_SIZE),aarch64-aarch64-32)
+    TARGET_ARCH = armv7l
+endif
+
+# operating system
+HOST_OS   := $(shell uname -s 2>/dev/null | tr "[:upper:]" "[:lower:]")
+TARGET_OS ?= $(HOST_OS)
+ifeq (,$(filter $(TARGET_OS),linux darwin qnx android))
+    $(error ERROR - unsupported value $(TARGET_OS) for TARGET_OS!)
+endif
+
+# host compiler
+ifeq ($(TARGET_OS),darwin)
+    ifeq ($(shell expr `xcodebuild -version | grep -i xcode | awk '{print $$2}' | cut -d'.' -f1` \>= 5),1)
+        HOST_COMPILER ?= clang++
+    endif
+else ifneq ($(TARGET_ARCH),$(HOST_ARCH))
+    ifeq ($(HOST_ARCH)-$(TARGET_ARCH),x86_64-armv7l)
+        ifeq ($(TARGET_OS),linux)
+            HOST_COMPILER ?= arm-linux-gnueabihf-g++
+        else ifeq ($(TARGET_OS),qnx)
+            ifeq ($(QNX_HOST),)
+                $(error ERROR - QNX_HOST must be passed to the QNX host toolchain)
+            endif
+            ifeq ($(QNX_TARGET),)
+                $(error ERROR - QNX_TARGET must be passed to the QNX target toolchain)
+            endif
+            export QNX_HOST
+            export QNX_TARGET
+            HOST_COMPILER ?= $(QNX_HOST)/usr/bin/arm-unknown-nto-qnx6.6.0eabi-g++
+        else ifeq ($(TARGET_OS),android)
+            HOST_COMPILER ?= arm-linux-androideabi-g++
+        endif
+    else ifeq ($(TARGET_ARCH),aarch64)
+        ifeq ($(TARGET_OS), linux)
+            HOST_COMPILER ?= aarch64-linux-gnu-g++
+        else ifeq ($(TARGET_OS),qnx)
+            ifeq ($(QNX_HOST),)
+                $(error ERROR - QNX_HOST must be passed to the QNX host toolchain)
+            endif
+            ifeq ($(QNX_TARGET),)
+                $(error ERROR - QNX_TARGET must be passed to the QNX target toolchain)
+            endif
+            export QNX_HOST
+            export QNX_TARGET
+            HOST_COMPILER ?= $(QNX_HOST)/usr/bin/q++
+        else ifeq ($(TARGET_OS), android)
+            HOST_COMPILER ?= aarch64-linux-android-clang++
+        endif
+    else ifeq ($(TARGET_ARCH),sbsa)
+        HOST_COMPILER ?= aarch64-linux-gnu-g++
+    else ifeq ($(TARGET_ARCH),ppc64le)
+        HOST_COMPILER ?= powerpc64le-linux-gnu-g++
+    endif
+endif
+HOST_COMPILER ?= g++
+NVCC          := $(CUDA_PATH)/bin/nvcc -ccbin $(HOST_COMPILER)
+
+# internal flags
+NVCCFLAGS   := -m${TARGET_SIZE}
+CCFLAGS     :=
+LDFLAGS     :=
+
+# build flags
+ifeq ($(TARGET_OS),darwin)
+    LDFLAGS += -rpath $(CUDA_PATH)/lib
+    CCFLAGS += -arch $(HOST_ARCH)
+else ifeq ($(HOST_ARCH)-$(TARGET_ARCH)-$(TARGET_OS),x86_64-armv7l-linux)
+    LDFLAGS += --dynamic-linker=/lib/ld-linux-armhf.so.3
+    CCFLAGS += -mfloat-abi=hard
+else ifeq ($(TARGET_OS),android)
+    LDFLAGS += -pie
+    CCFLAGS += -fpie -fpic -fexceptions
+endif
+
+ifneq ($(TARGET_ARCH),$(HOST_ARCH))
+    ifeq ($(TARGET_ARCH)-$(TARGET_OS),armv7l-linux)
+        ifneq ($(TARGET_FS),)
+            GCCVERSIONLTEQ46 := $(shell expr `$(HOST_COMPILER) -dumpversion` \<= 4.6)
+            ifeq ($(GCCVERSIONLTEQ46),1)
+                CCFLAGS += --sysroot=$(TARGET_FS)
+            endif
+            LDFLAGS += --sysroot=$(TARGET_FS)
+            LDFLAGS += -rpath-link=$(TARGET_FS)/lib
+            LDFLAGS += -rpath-link=$(TARGET_FS)/usr/lib
+            LDFLAGS += -rpath-link=$(TARGET_FS)/usr/lib/arm-linux-gnueabihf
+        endif
+    endif
+    ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-linux)
+        ifneq ($(TARGET_FS),)
+            GCCVERSIONLTEQ46 := $(shell expr `$(HOST_COMPILER) -dumpversion` \<= 4.6)
+            ifeq ($(GCCVERSIONLTEQ46),1)
+                CCFLAGS += --sysroot=$(TARGET_FS)
+            endif
+            LDFLAGS += --sysroot=$(TARGET_FS)
+            LDFLAGS += -rpath-link=$(TARGET_FS)/lib -L$(TARGET_FS)/lib
+            LDFLAGS += -rpath-link=$(TARGET_FS)/lib/aarch64-linux-gnu -L$(TARGET_FS)/lib/aarch64-linux-gnu
+            LDFLAGS += -rpath-link=$(TARGET_FS)/usr/lib -L$(TARGET_FS)/usr/lib
+            LDFLAGS += -rpath-link=$(TARGET_FS)/usr/lib/aarch64-linux-gnu -L$(TARGET_FS)/usr/lib/aarch64-linux-gnu
+            LDFLAGS += --unresolved-symbols=ignore-in-shared-libs
+            CCFLAGS += -isystem=$(TARGET_FS)/usr/include -I$(TARGET_FS)/usr/include -I$(TARGET_FS)/usr/include/libdrm
+            CCFLAGS += -isystem=$(TARGET_FS)/usr/include/aarch64-linux-gnu -I$(TARGET_FS)/usr/include/aarch64-linux-gnu
+        endif
+    endif
+    ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-qnx)
+        NVCCFLAGS += -D_QNX_SOURCE
+        NVCCFLAGS += --qpp-config 8.3.0,gcc_ntoaarch64le
+        CCFLAGS += -DWIN_INTERFACE_CUSTOM -I/usr/include/aarch64-qnx-gnu
+        LDFLAGS += -lsocket
+        LDFLAGS += -L/usr/lib/aarch64-qnx-gnu
+        CCFLAGS += "-Wl\,-rpath-link\,/usr/lib/aarch64-qnx-gnu"
+        ifdef TARGET_OVERRIDE
+            LDFLAGS += -lslog2
+        endif
+
+        ifneq ($(TARGET_FS),)
+            LDFLAGS += -L$(TARGET_FS)/usr/lib
+            CCFLAGS += "-Wl\,-rpath-link\,$(TARGET_FS)/usr/lib"
+            LDFLAGS += -L$(TARGET_FS)/usr/libnvidia
+            CCFLAGS += "-Wl\,-rpath-link\,$(TARGET_FS)/usr/libnvidia"
+            CCFLAGS += -I$(TARGET_FS)/../include
+        endif
+    endif
+endif
+
+ifdef TARGET_OVERRIDE # cuda toolkit targets override
+    NVCCFLAGS += -target-dir $(TARGET_OVERRIDE)
+endif
+
+# Install directory of different arch
+CUDA_INSTALL_TARGET_DIR :=
+ifeq ($(TARGET_ARCH)-$(TARGET_OS),armv7l-linux)
+    CUDA_INSTALL_TARGET_DIR = targets/armv7-linux-gnueabihf/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-linux)
+    CUDA_INSTALL_TARGET_DIR = targets/aarch64-linux/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),sbsa-linux)
+    CUDA_INSTALL_TARGET_DIR = targets/sbsa-linux/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),armv7l-android)
+    CUDA_INSTALL_TARGET_DIR = targets/armv7-linux-androideabi/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-android)
+    CUDA_INSTALL_TARGET_DIR = targets/aarch64-linux-androideabi/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),armv7l-qnx)
+    CUDA_INSTALL_TARGET_DIR = targets/ARMv7-linux-QNX/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-qnx)
+    CUDA_INSTALL_TARGET_DIR = targets/aarch64-qnx/
+else ifeq ($(TARGET_ARCH),ppc64le)
+    CUDA_INSTALL_TARGET_DIR = targets/ppc64le-linux/
+endif
+
+# Debug build flags
+ifeq ($(dbg),1)
+      NVCCFLAGS += -g -G
+      BUILD_TYPE := debug
+else
+      BUILD_TYPE := release
+endif
+
+ALL_CCFLAGS :=
+ALL_CCFLAGS += $(NVCCFLAGS)
+ALL_CCFLAGS += $(EXTRA_NVCCFLAGS)
+ALL_CCFLAGS += $(addprefix -Xcompiler ,$(CCFLAGS))
+ALL_CCFLAGS += $(addprefix -Xcompiler ,$(EXTRA_CCFLAGS))
+
+SAMPLE_ENABLED := 1
+
+# This sample is not supported on QNX
+ifeq ($(TARGET_OS),qnx)
+  $(info >>> WARNING - nbody is not supported on QNX - waiving sample <<<)
+  SAMPLE_ENABLED := 0
+endif
+
+ALL_LDFLAGS :=
+ALL_LDFLAGS += $(ALL_CCFLAGS)
+ALL_LDFLAGS += $(addprefix -Xlinker ,$(LDFLAGS))
+ALL_LDFLAGS += $(addprefix -Xlinker ,$(EXTRA_LDFLAGS))
+
+# Common includes and paths for CUDA
+INCLUDES  := -I../../../Common
+LIBRARIES :=
+
+################################################################################
+
+# Makefile include to help find GL Libraries
+include ./findgllib.mk
+
+# OpenGL specific libraries
+ifeq ($(TARGET_OS),darwin)
+ # Mac OSX specific libraries and paths to include
+ LIBRARIES += -L/System/Library/Frameworks/OpenGL.framework/Libraries
+ LIBRARIES += -lGL -lGLU
+ ALL_LDFLAGS += -Xlinker -framework -Xlinker GLUT
+else
+ LIBRARIES += $(GLLINK)
+ LIBRARIES += -lGL -lGLU -lglut
+endif
+
+# Gencode arguments
+ifeq ($(TARGET_ARCH),$(filter $(TARGET_ARCH),armv7l aarch64 sbsa))
+SMS ?= 53 61 70 72 75 80 86 87
+else
+SMS ?= 35 37 50 52 60 61 70 75 80 86
+endif
+
+ifeq ($(SMS),)
+$(info >>> WARNING - no SM architectures have been specified - waiving sample <<<)
+SAMPLE_ENABLED := 0
+endif
+
+ifeq ($(GENCODE_FLAGS),)
+# Generate SASS code for each SM architecture listed in $(SMS)
+$(foreach sm,$(SMS),$(eval GENCODE_FLAGS += -gencode arch=compute_$(sm),code=sm_$(sm)))
+
+# Generate PTX code from the highest SM architecture in $(SMS) to guarantee forward-compatibility
+HIGHEST_SM := $(lastword $(sort $(SMS)))
+ifneq ($(HIGHEST_SM),)
+GENCODE_FLAGS += -gencode arch=compute_$(HIGHEST_SM),code=compute_$(HIGHEST_SM)
+endif
+endif
+
+ALL_CCFLAGS += -ftz=true --threads 0 --std=c++11
+
+ifeq ($(SAMPLE_ENABLED),0)
+EXEC ?= @echo "[@]"
+endif
+
+################################################################################
+
+# Target rules
+all: build
+
+build: nbody
+
+check.deps:
+ifeq ($(SAMPLE_ENABLED),0)
+	@echo "Sample will be waived due to the above missing dependencies"
+else
+	@echo "Sample is ready - all dependencies have been met"
+endif
+
+bodysystemcuda.o:bodysystemcuda.cu
+	$(EXEC) $(NVCC) $(INCLUDES) $(ALL_CCFLAGS) $(GENCODE_FLAGS) -o $@ -c $<
+
+nbody.o:nbody.cpp
+	$(EXEC) $(NVCC) $(INCLUDES) $(ALL_CCFLAGS) $(GENCODE_FLAGS) -o $@ -c $<
+
+render_particles.o:render_particles.cpp
+	$(EXEC) $(NVCC) $(INCLUDES) $(ALL_CCFLAGS) $(GENCODE_FLAGS) -o $@ -c $<
+
+nbody: bodysystemcuda.o nbody.o render_particles.o
+	$(EXEC) $(NVCC) $(ALL_LDFLAGS) $(GENCODE_FLAGS) -o $@ $+ $(LIBRARIES)
+	$(EXEC) mkdir -p ../../../bin/$(TARGET_ARCH)/$(TARGET_OS)/$(BUILD_TYPE)
+	$(EXEC) cp $@ ../../../bin/$(TARGET_ARCH)/$(TARGET_OS)/$(BUILD_TYPE)
+
+run: build
+	$(EXEC) ./nbody
+
+testrun: build
+	$(EXEC) ./nbody -benchmark -compare -cpu
+
+clean:
+	rm -f nbody bodysystemcuda.o nbody.o render_particles.o
+	rm -rf ../../../bin/$(TARGET_ARCH)/$(TARGET_OS)/$(BUILD_TYPE)/nbody
+
+clobber: clean

Samples/5_Domain_Specific/nbody/NsightEclipse.xml

@@ -0,0 +1,119 @@
+<?xml version="1.0" encoding="UTF-8"?> 
+<!DOCTYPE entry SYSTEM "SamplesInfo.dtd">
+<entry>
+  <name>nbody</name>
+  <cflags>
+    <flag>-ftz=true</flag>
+  </cflags>
+  <cuda_api_list>
+    <toolkit>cudaMemcpyToSymbol</toolkit>
+    <toolkit>cudaGraphicsMapResources</toolkit>
+    <toolkit>cudaEventRecord</toolkit>
+    <toolkit>cudaStreamQuery</toolkit>
+    <toolkit>cudaEventCreate</toolkit>
+    <toolkit>cudaGraphicsResourceGetMappedPointer</toolkit>
+    <toolkit>cudaGetDeviceCount</toolkit>
+    <toolkit>cudaEventElapsedTime</toolkit>
+    <toolkit>cudaDeviceSynchronize</toolkit>
+    <toolkit>cudaEventSynchronize</toolkit>
+    <toolkit>cudaGraphicsResourceSetMapFlags</toolkit>
+    <toolkit>cudaSetDeviceFlags</toolkit>
+    <toolkit>cudaEventDestroy</toolkit>
+    <toolkit>cudaDeviceCanAccessPeer</toolkit>
+    <toolkit>cudaSetDevice</toolkit>
+    <toolkit>cudaGraphicsUnmapResources</toolkit>
+    <toolkit>cudaGetDeviceProperties</toolkit>
+    <toolkit>cudaGetDevice</toolkit>
+  </cuda_api_list>
+  <description><![CDATA[This sample demonstrates efficient all-pairs simulation of a gravitational n-body simulation in CUDA.  This sample accompanies the GPU Gems 3 chapter "Fast N-Body Simulation with CUDA".  With CUDA 5.5, performance on Tesla K20c has increased to over 1.8TFLOP/s single precision.  Double Performance has also improved on all Kepler and Fermi GPU architectures as well.  Starting in CUDA 4.0, the nBody sample has been updated to take advantage of new features to easily scale the n-body simulation across multiple GPUs in a single PC.  Adding "-numbodies=<bodies>" to the command line will allow users to set # of bodies for simulation.  Adding “-numdevices=<N>” to the command line option will cause the sample to use N devices (if available) for simulation.  In this mode, the position and velocity data for all bodies are read from system memory using “zero copy” rather than from device memory.  For a small number of devices (4 or fewer) and a large enough number of bodies, bandwidth is not a bottleneck so we can achieve strong scaling across these devices.]]></description>
+  <devicecompilation>whole</devicecompilation>
+  <files>
+    <file>./galaxy_20K.bin</file>
+    <file>./CMakeLists.txt</file>
+  </files>
+  <includepaths>
+    <path>./</path>
+    <path>../</path>
+    <path>../../../Common</path>
+  </includepaths>
+  <keyconcepts>
+    <concept level="advanced">Graphics Interop</concept>
+    <concept level="advanced">Data Parallel Algorithms</concept>
+    <concept level="advanced">Physically-Based Simulation</concept>
+  </keyconcepts>
+  <keywords>
+    <keyword>CUDA</keyword>
+    <keyword>GPGPU</keyword>
+    <keyword>n-body</keyword>
+    <keyword>simulation</keyword>
+    <keyword>astrophysics</keyword>
+    <keyword>opengl</keyword>
+    <keyword>openGL</keyword>
+  </keywords>
+  <libraries>
+    <library>GLU</library>
+    <library>GL</library>
+    <library framework="true" os="macosx">GLUT</library>
+    <library os="linux">GLEW</library>
+    <library os="linux">glut</library>
+    <library os="linux">X11</library>
+  </libraries>
+  <librarypaths>
+    <path arch="x86_64" os="linux">../../../common/lib/linux/x86_64</path>
+    <path arch="armv7l" os="linux">../../../common/lib/linux/armv7l</path>
+    <path os="macosx">../../../common/lib/darwin</path>
+  </librarypaths>
+  <nsight_eclipse>true</nsight_eclipse>
+  <primary_file>nbody.cpp</primary_file>
+  <qatests>
+    <qatest>-benchmark -compare -cpu</qatest>
+  </qatests>
+  <required_dependencies>
+    <dependency>X11</dependency>
+    <dependency>GL</dependency>
+  </required_dependencies>
+  <scopes>
+    <scope>2:Graphics Interop</scope>
+    <scope>1:CUDA Advanced Topics</scope>
+    <scope>1:Data-Parallel Algorithms</scope>
+    <scope>3:Physically-Based Simulation</scope>
+  </scopes>
+  <sm-arch>sm35</sm-arch>
+  <sm-arch>sm37</sm-arch>
+  <sm-arch>sm50</sm-arch>
+  <sm-arch>sm52</sm-arch>
+  <sm-arch>sm53</sm-arch>
+  <sm-arch>sm60</sm-arch>
+  <sm-arch>sm61</sm-arch>
+  <sm-arch>sm70</sm-arch>
+  <sm-arch>sm72</sm-arch>
+  <sm-arch>sm75</sm-arch>
+  <sm-arch>sm80</sm-arch>
+  <sm-arch>sm86</sm-arch>
+  <sm-arch>sm87</sm-arch>
+  <supported_envs>
+    <env>
+      <arch>x86_64</arch>
+      <platform>linux</platform>
+    </env>
+    <env>
+      <platform>windows7</platform>
+    </env>
+    <env>
+      <arch>x86_64</arch>
+      <platform>macosx</platform>
+    </env>
+    <env>
+      <arch>arm</arch>
+    </env>
+    <env>
+      <arch>sbsa</arch>
+    </env>
+  </supported_envs>
+  <supported_sm_architectures>
+    <include>all</include>
+  </supported_sm_architectures>
+  <title>CUDA N-Body Simulation</title>
+  <type>exe</type>
+  <whitepaper>doc/nbody_gems3_ch31.pdf</whitepaper>
+</entry>

Samples/5_Domain_Specific/nbody/README.md

@@ -0,0 +1,75 @@
+# nbody - CUDA N-Body Simulation
+
+## Description
+
+This sample demonstrates efficient all-pairs simulation of a gravitational n-body simulation in CUDA.  This sample accompanies the GPU Gems 3 chapter "Fast N-Body Simulation with CUDA".  With CUDA 5.5, performance on Tesla K20c has increased to over 1.8TFLOP/s single precision.  Double Performance has also improved on all Kepler and Fermi GPU architectures as well.  Starting in CUDA 4.0, the nBody sample has been updated to take advantage of new features to easily scale the n-body simulation across multiple GPUs in a single PC.  Adding "-numbodies=<bodies>" to the command line will allow users to set # of bodies for simulation.  Adding “-numdevices=<N>” to the command line option will cause the sample to use N devices (if available) for simulation.  In this mode, the position and velocity data for all bodies are read from system memory using “zero copy” rather than from device memory.  For a small number of devices (4 or fewer) and a large enough number of bodies, bandwidth is not a bottleneck so we can achieve strong scaling across these devices.
+
+## Key Concepts
+
+Graphics Interop, Data Parallel Algorithms, Physically-Based Simulation
+
+## Supported SM Architectures
+
+[SM 3.5 ](https://developer.nvidia.com/cuda-gpus)  [SM 3.7 ](https://developer.nvidia.com/cuda-gpus)  [SM 5.0 ](https://developer.nvidia.com/cuda-gpus)  [SM 5.2 ](https://developer.nvidia.com/cuda-gpus)  [SM 5.3 ](https://developer.nvidia.com/cuda-gpus)  [SM 6.0 ](https://developer.nvidia.com/cuda-gpus)  [SM 6.1 ](https://developer.nvidia.com/cuda-gpus)  [SM 7.0 ](https://developer.nvidia.com/cuda-gpus)  [SM 7.2 ](https://developer.nvidia.com/cuda-gpus)  [SM 7.5 ](https://developer.nvidia.com/cuda-gpus)  [SM 8.0 ](https://developer.nvidia.com/cuda-gpus)  [SM 8.6 ](https://developer.nvidia.com/cuda-gpus)  [SM 8.7 ](https://developer.nvidia.com/cuda-gpus)
+
+## Supported OSes
+
+Linux, Windows
+
+## Supported CPU Architecture
+
+x86_64, armv7l
+
+## CUDA APIs involved
+
+### [CUDA Runtime API](http://docs.nvidia.com/cuda/cuda-runtime-api/index.html)
+cudaMemcpyToSymbol, cudaGraphicsMapResources, cudaEventRecord, cudaStreamQuery, cudaEventCreate, cudaGraphicsResourceGetMappedPointer, cudaGetDeviceCount, cudaEventElapsedTime, cudaDeviceSynchronize, cudaEventSynchronize, cudaGraphicsResourceSetMapFlags, cudaSetDeviceFlags, cudaEventDestroy, cudaDeviceCanAccessPeer, cudaSetDevice, cudaGraphicsUnmapResources, cudaGetDeviceProperties, cudaGetDevice
+
+## Dependencies needed to build/run
+[X11](../../README.md#x11), [GL](../../README.md#gl)
+
+## Prerequisites
+
+Download and install the [CUDA Toolkit 11.6](https://developer.nvidia.com/cuda-downloads) for your corresponding platform.
+Make sure the dependencies mentioned in [Dependencies]() section above are installed.
+
+## Build and Run
+
+### Windows
+The Windows samples are built using the Visual Studio IDE. Solution files (.sln) are provided for each supported version of Visual Studio, using the format:
+```
+*_vs<version>.sln - for Visual Studio <version>
+```
+Each individual sample has its own set of solution files in its directory:
+
+To build/examine all the samples at once, the complete solution files should be used. To build/examine a single sample, the individual sample solution files should be used.
+> **Note:** Some samples require that the Microsoft DirectX SDK (June 2010 or newer) be installed and that the VC++ directory paths are properly set up (**Tools > Options...**). Check DirectX Dependencies section for details."
+
+### Linux
+The Linux samples are built using makefiles. To use the makefiles, change the current directory to the sample directory you wish to build, and run make:
+```
+$ cd <sample_dir>
+$ make
+```
+The samples makefiles can take advantage of certain options:
+*  **TARGET_ARCH=<arch>** - cross-compile targeting a specific architecture. Allowed architectures are x86_64, armv7l.
+    By default, TARGET_ARCH is set to HOST_ARCH. On a x86_64 machine, not setting TARGET_ARCH is the equivalent of setting TARGET_ARCH=x86_64.<br/>
+`$ make TARGET_ARCH=x86_64` <br/> `$ make TARGET_ARCH=armv7l` <br/>
+    See [here](http://docs.nvidia.com/cuda/cuda-samples/index.html#cross-samples) for more details.
+*   **dbg=1** - build with debug symbols
+    ```
+    $ make dbg=1
+    ```
+*   **SMS="A B ..."** - override the SM architectures for which the sample will be built, where `"A B ..."` is a space-delimited list of SM architectures. For example, to generate SASS for SM 50 and SM 60, use `SMS="50 60"`.
+    ```
+    $ make SMS="50 60"
+    ```
+
+*  **HOST_COMPILER=<host_compiler>** - override the default g++ host compiler. See the [Linux Installation Guide](http://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html#system-requirements) for a list of supported host compilers.
+```
+    $ make HOST_COMPILER=g++
+```
+
+## References (for more details)
+
+[whitepaper](./doc/nbody_gems3_ch31.pdf)

Samples/5_Domain_Specific/nbody/bodysystem.h

@@ -0,0 +1,286 @@
+/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __BODYSYSTEM_H__
+#define __BODYSYSTEM_H__
+
+#include <algorithm>
+
+enum NBodyConfig {
+  NBODY_CONFIG_RANDOM,
+  NBODY_CONFIG_SHELL,
+  NBODY_CONFIG_EXPAND,
+  NBODY_NUM_CONFIGS
+};
+
+enum BodyArray {
+  BODYSYSTEM_POSITION,
+  BODYSYSTEM_VELOCITY,
+};
+
+template <typename T>
+struct vec3 {
+  typedef float Type;
+};  // dummy
+template <>
+struct vec3<float> {
+  typedef float3 Type;
+};
+template <>
+struct vec3<double> {
+  typedef double3 Type;
+};
+
+template <typename T>
+struct vec4 {
+  typedef float Type;
+};  // dummy
+template <>
+struct vec4<float> {
+  typedef float4 Type;
+};
+template <>
+struct vec4<double> {
+  typedef double4 Type;
+};
+
+class string;
+
+// BodySystem abstract base class
+template <typename T>
+class BodySystem {
+ public:  // methods
+  BodySystem(int numBodies) {}
+  virtual ~BodySystem() {}
+
+  virtual void loadTipsyFile(const std::string &filename) = 0;
+
+  virtual void update(T deltaTime) = 0;
+
+  virtual void setSoftening(T softening) = 0;
+  virtual void setDamping(T damping) = 0;
+
+  virtual T *getArray(BodyArray array) = 0;
+  virtual void setArray(BodyArray array, const T *data) = 0;
+
+  virtual unsigned int getCurrentReadBuffer() const = 0;
+
+  virtual unsigned int getNumBodies() const = 0;
+
+  virtual void synchronizeThreads() const {};
+
+ protected:        // methods
+  BodySystem() {}  // default constructor
+
+  virtual void _initialize(int numBodies) = 0;
+  virtual void _finalize() = 0;
+};
+
+inline float3 scalevec(float3 &vector, float scalar) {
+  float3 rt = vector;
+  rt.x *= scalar;
+  rt.y *= scalar;
+  rt.z *= scalar;
+  return rt;
+}
+
+inline float normalize(float3 &vector) {
+  float dist =
+      sqrtf(vector.x * vector.x + vector.y * vector.y + vector.z * vector.z);
+
+  if (dist > 1e-6) {
+    vector.x /= dist;
+    vector.y /= dist;
+    vector.z /= dist;
+  }
+
+  return dist;
+}
+
+inline float dot(float3 v0, float3 v1) {
+  return v0.x * v1.x + v0.y * v1.y + v0.z * v1.z;
+}
+
+inline float3 cross(float3 v0, float3 v1) {
+  float3 rt;
+  rt.x = v0.y * v1.z - v0.z * v1.y;
+  rt.y = v0.z * v1.x - v0.x * v1.z;
+  rt.z = v0.x * v1.y - v0.y * v1.x;
+  return rt;
+}
+
+// utility function
+template <typename T>
+void randomizeBodies(NBodyConfig config, T *pos, T *vel, float *color,
+                     float clusterScale, float velocityScale, int numBodies,
+                     bool vec4vel) {
+  switch (config) {
+    default:
+    case NBODY_CONFIG_RANDOM: {
+      float scale = clusterScale * std::max<float>(1.0f, numBodies / (1024.0f));
+      float vscale = velocityScale * scale;
+
+      int p = 0, v = 0;
+      int i = 0;
+
+      while (i < numBodies) {
+        float3 point;
+        // const int scale = 16;
+        point.x = rand() / (float)RAND_MAX * 2 - 1;
+        point.y = rand() / (float)RAND_MAX * 2 - 1;
+        point.z = rand() / (float)RAND_MAX * 2 - 1;
+        float lenSqr = dot(point, point);
+
+        if (lenSqr > 1) continue;
+
+        float3 velocity;
+        velocity.x = rand() / (float)RAND_MAX * 2 - 1;
+        velocity.y = rand() / (float)RAND_MAX * 2 - 1;
+        velocity.z = rand() / (float)RAND_MAX * 2 - 1;
+        lenSqr = dot(velocity, velocity);
+
+        if (lenSqr > 1) continue;
+
+        pos[p++] = point.x * scale;  // pos.x
+        pos[p++] = point.y * scale;  // pos.y
+        pos[p++] = point.z * scale;  // pos.z
+        pos[p++] = 1.0f;             // mass
+
+        vel[v++] = velocity.x * vscale;  // pos.x
+        vel[v++] = velocity.y * vscale;  // pos.x
+        vel[v++] = velocity.z * vscale;  // pos.x
+
+        if (vec4vel) vel[v++] = 1.0f;  // inverse mass
+
+        i++;
+      }
+    } break;
+
+    case NBODY_CONFIG_SHELL: {
+      float scale = clusterScale;
+      float vscale = scale * velocityScale;
+      float inner = 2.5f * scale;
+      float outer = 4.0f * scale;
+
+      int p = 0, v = 0;
+      int i = 0;
+
+      while (i < numBodies)  // for(int i=0; i < numBodies; i++)
+      {
+        float x, y, z;
+        x = rand() / (float)RAND_MAX * 2 - 1;
+        y = rand() / (float)RAND_MAX * 2 - 1;
+        z = rand() / (float)RAND_MAX * 2 - 1;
+
+        float3 point = {x, y, z};
+        float len = normalize(point);
+
+        if (len > 1) continue;
+
+        pos[p++] =
+            point.x * (inner + (outer - inner) * rand() / (float)RAND_MAX);
+        pos[p++] =
+            point.y * (inner + (outer - inner) * rand() / (float)RAND_MAX);
+        pos[p++] =
+            point.z * (inner + (outer - inner) * rand() / (float)RAND_MAX);
+        pos[p++] = 1.0f;
+
+        x = 0.0f;  // * (rand() / (float) RAND_MAX * 2 - 1);
+        y = 0.0f;  // * (rand() / (float) RAND_MAX * 2 - 1);
+        z = 1.0f;  // * (rand() / (float) RAND_MAX * 2 - 1);
+        float3 axis = {x, y, z};
+        normalize(axis);
+
+        if (1 - dot(point, axis) < 1e-6) {
+          axis.x = point.y;
+          axis.y = point.x;
+          normalize(axis);
+        }
+
+        // if (point.y < 0) axis = scalevec(axis, -1);
+        float3 vv = {(float)pos[4 * i], (float)pos[4 * i + 1],
+                     (float)pos[4 * i + 2]};
+        vv = cross(vv, axis);
+        vel[v++] = vv.x * vscale;
+        vel[v++] = vv.y * vscale;
+        vel[v++] = vv.z * vscale;
+
+        if (vec4vel) vel[v++] = 1.0f;
+
+        i++;
+      }
+    } break;
+
+    case NBODY_CONFIG_EXPAND: {
+      float scale = clusterScale * numBodies / (1024.f);
+
+      if (scale < 1.0f) scale = clusterScale;
+
+      float vscale = scale * velocityScale;
+
+      int p = 0, v = 0;
+
+      for (int i = 0; i < numBodies;) {
+        float3 point;
+
+        point.x = rand() / (float)RAND_MAX * 2 - 1;
+        point.y = rand() / (float)RAND_MAX * 2 - 1;
+        point.z = rand() / (float)RAND_MAX * 2 - 1;
+
+        float lenSqr = dot(point, point);
+
+        if (lenSqr > 1) continue;
+
+        pos[p++] = point.x * scale;   // pos.x
+        pos[p++] = point.y * scale;   // pos.y
+        pos[p++] = point.z * scale;   // pos.z
+        pos[p++] = 1.0f;              // mass
+        vel[v++] = point.x * vscale;  // pos.x
+        vel[v++] = point.y * vscale;  // pos.x
+        vel[v++] = point.z * vscale;  // pos.x
+
+        if (vec4vel) vel[v++] = 1.0f;  // inverse mass
+
+        i++;
+      }
+    } break;
+  }
+
+  if (color) {
+    int v = 0;
+
+    for (int i = 0; i < numBodies; i++) {
+      // const int scale = 16;
+      color[v++] = rand() / (float)RAND_MAX;
+      color[v++] = rand() / (float)RAND_MAX;
+      color[v++] = rand() / (float)RAND_MAX;
+      color[v++] = 1.0f;
+    }
+  }
+}
+
+#endif  // __BODYSYSTEM_H__

Samples/5_Domain_Specific/nbody/bodysystemcpu.h

@@ -0,0 +1,79 @@
+/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __BODYSYSTEMCPU_H__
+#define __BODYSYSTEMCPU_H__
+
+#include "bodysystem.h"
+
+// CPU Body System
+template <typename T>
+class BodySystemCPU : public BodySystem<T> {
+ public:
+  BodySystemCPU(int numBodies);
+  virtual ~BodySystemCPU();
+
+  virtual void loadTipsyFile(const std::string &filename);
+
+  virtual void update(T deltaTime);
+
+  virtual void setSoftening(T softening) {
+    m_softeningSquared = softening * softening;
+  }
+  virtual void setDamping(T damping) { m_damping = damping; }
+
+  virtual T *getArray(BodyArray array);
+  virtual void setArray(BodyArray array, const T *data);
+
+  virtual unsigned int getCurrentReadBuffer() const { return 0; }
+
+  virtual unsigned int getNumBodies() const { return m_numBodies; }
+
+ protected:           // methods
+  BodySystemCPU() {}  // default constructor
+
+  virtual void _initialize(int numBodies);
+  virtual void _finalize();
+
+  void _computeNBodyGravitation();
+  void _integrateNBodySystem(T deltaTime);
+
+ protected:  // data
+  int m_numBodies;
+  bool m_bInitialized;
+
+  T *m_pos;
+  T *m_vel;
+  T *m_force;
+
+  T m_softeningSquared;
+  T m_damping;
+};
+
+#include "bodysystemcpu_impl.h"
+
+#endif  // __BODYSYSTEMCPU_H__

Samples/5_Domain_Specific/nbody/bodysystemcpu_impl.h

@@ -0,0 +1,280 @@
+/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "bodysystemcpu.h"
+
+#include <assert.h>
+#include <memory.h>
+#include <math.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <helper_cuda.h>
+#include <algorithm>
+#include "tipsy.h"
+
+#ifdef OPENMP
+#include <omp.h>
+#endif
+
+template <typename T>
+BodySystemCPU<T>::BodySystemCPU(int numBodies)
+    : m_numBodies(numBodies),
+      m_bInitialized(false),
+      m_force(0),
+      m_softeningSquared(.00125f),
+      m_damping(0.995f) {
+  m_pos = 0;
+  m_vel = 0;
+
+  _initialize(numBodies);
+}
+
+template <typename T>
+BodySystemCPU<T>::~BodySystemCPU() {
+  _finalize();
+  m_numBodies = 0;
+}
+
+template <typename T>
+void BodySystemCPU<T>::_initialize(int numBodies) {
+  assert(!m_bInitialized);
+
+  m_numBodies = numBodies;
+
+  m_pos = new T[m_numBodies * 4];
+  m_vel = new T[m_numBodies * 4];
+  m_force = new T[m_numBodies * 3];
+
+  memset(m_pos, 0, m_numBodies * 4 * sizeof(T));
+  memset(m_vel, 0, m_numBodies * 4 * sizeof(T));
+  memset(m_force, 0, m_numBodies * 3 * sizeof(T));
+
+  m_bInitialized = true;
+}
+
+template <typename T>
+void BodySystemCPU<T>::_finalize() {
+  assert(m_bInitialized);
+
+  delete[] m_pos;
+  delete[] m_vel;
+  delete[] m_force;
+
+  m_bInitialized = false;
+}
+
+template <typename T>
+void BodySystemCPU<T>::loadTipsyFile(const std::string &filename) {
+  if (m_bInitialized) _finalize();
+
+  vector<typename vec4<T>::Type> positions;
+  vector<typename vec4<T>::Type> velocities;
+  vector<int> ids;
+
+  int nBodies = 0;
+  int nFirst = 0, nSecond = 0, nThird = 0;
+
+  read_tipsy_file(positions, velocities, ids, filename, nBodies, nFirst,
+                  nSecond, nThird);
+
+  _initialize(nBodies);
+
+  memcpy(m_pos, &positions[0], sizeof(vec4<T>) * nBodies);
+  memcpy(m_vel, &velocities[0], sizeof(vec4<T>) * nBodies);
+}
+
+template <typename T>
+void BodySystemCPU<T>::update(T deltaTime) {
+  assert(m_bInitialized);
+
+  _integrateNBodySystem(deltaTime);
+
+  // std::swap(m_currentRead, m_currentWrite);
+}
+
+template <typename T>
+T *BodySystemCPU<T>::getArray(BodyArray array) {
+  assert(m_bInitialized);
+
+  T *data = 0;
+
+  switch (array) {
+    default:
+    case BODYSYSTEM_POSITION:
+      data = m_pos;
+      break;
+
+    case BODYSYSTEM_VELOCITY:
+      data = m_vel;
+      break;
+  }
+
+  return data;
+}
+
+template <typename T>
+void BodySystemCPU<T>::setArray(BodyArray array, const T *data) {
+  assert(m_bInitialized);
+
+  T *target = 0;
+
+  switch (array) {
+    default:
+    case BODYSYSTEM_POSITION:
+      target = m_pos;
+      break;
+
+    case BODYSYSTEM_VELOCITY:
+      target = m_vel;
+      break;
+  }
+
+  memcpy(target, data, m_numBodies * 4 * sizeof(T));
+}
+
+template <typename T>
+T sqrt_T(T x) {
+  return sqrt(x);
+}
+
+template <>
+float sqrt_T<float>(float x) {
+  return sqrtf(x);
+}
+
+template <typename T>
+void bodyBodyInteraction(T accel[3], T posMass0[4], T posMass1[4],
+                         T softeningSquared) {
+  T r[3];
+
+  // r_01  [3 FLOPS]
+  r[0] = posMass1[0] - posMass0[0];
+  r[1] = posMass1[1] - posMass0[1];
+  r[2] = posMass1[2] - posMass0[2];
+
+  // d^2 + e^2 [6 FLOPS]
+  T distSqr = r[0] * r[0] + r[1] * r[1] + r[2] * r[2];
+  distSqr += softeningSquared;
+
+  // invDistCube =1/distSqr^(3/2)  [4 FLOPS (2 mul, 1 sqrt, 1 inv)]
+  T invDist = (T)1.0 / (T)sqrt((double)distSqr);
+  T invDistCube = invDist * invDist * invDist;
+
+  // s = m_j * invDistCube [1 FLOP]
+  T s = posMass1[3] * invDistCube;
+
+  // (m_1 * r_01) / (d^2 + e^2)^(3/2)  [6 FLOPS]
+  accel[0] += r[0] * s;
+  accel[1] += r[1] * s;
+  accel[2] += r[2] * s;
+}
+
+template <typename T>
+void BodySystemCPU<T>::_computeNBodyGravitation() {
+#ifdef OPENMP
+#pragma omp parallel for
+#endif
+
+  for (int i = 0; i < m_numBodies; i++) {
+    int indexForce = 3 * i;
+
+    T acc[3] = {0, 0, 0};
+
+    // We unroll this loop 4X for a small performance boost.
+    int j = 0;
+
+    while (j < m_numBodies) {
+      bodyBodyInteraction<T>(acc, &m_pos[4 * i], &m_pos[4 * j],
+                             m_softeningSquared);
+      j++;
+      bodyBodyInteraction<T>(acc, &m_pos[4 * i], &m_pos[4 * j],
+                             m_softeningSquared);
+      j++;
+      bodyBodyInteraction<T>(acc, &m_pos[4 * i], &m_pos[4 * j],
+                             m_softeningSquared);
+      j++;
+      bodyBodyInteraction<T>(acc, &m_pos[4 * i], &m_pos[4 * j],
+                             m_softeningSquared);
+      j++;
+    }
+
+    m_force[indexForce] = acc[0];
+    m_force[indexForce + 1] = acc[1];
+    m_force[indexForce + 2] = acc[2];
+  }
+}
+
+template <typename T>
+void BodySystemCPU<T>::_integrateNBodySystem(T deltaTime) {
+  _computeNBodyGravitation();
+
+#ifdef OPENMP
+#pragma omp parallel for
+#endif
+
+  for (int i = 0; i < m_numBodies; ++i) {
+    int index = 4 * i;
+    int indexForce = 3 * i;
+
+    T pos[3], vel[3], force[3];
+    pos[0] = m_pos[index + 0];
+    pos[1] = m_pos[index + 1];
+    pos[2] = m_pos[index + 2];
+    T invMass = m_pos[index + 3];
+
+    vel[0] = m_vel[index + 0];
+    vel[1] = m_vel[index + 1];
+    vel[2] = m_vel[index + 2];
+
+    force[0] = m_force[indexForce + 0];
+    force[1] = m_force[indexForce + 1];
+    force[2] = m_force[indexForce + 2];
+
+    // acceleration = force / mass;
+    // new velocity = old velocity + acceleration * deltaTime
+    vel[0] += (force[0] * invMass) * deltaTime;
+    vel[1] += (force[1] * invMass) * deltaTime;
+    vel[2] += (force[2] * invMass) * deltaTime;
+
+    vel[0] *= m_damping;
+    vel[1] *= m_damping;
+    vel[2] *= m_damping;
+
+    // new position = old position + velocity * deltaTime
+    pos[0] += vel[0] * deltaTime;
+    pos[1] += vel[1] * deltaTime;
+    pos[2] += vel[2] * deltaTime;
+
+    m_pos[index + 0] = pos[0];
+    m_pos[index + 1] = pos[1];
+    m_pos[index + 2] = pos[2];
+
+    m_vel[index + 0] = vel[0];
+    m_vel[index + 1] = vel[1];
+    m_vel[index + 2] = vel[2];
+  }
+}

Samples/5_Domain_Specific/nbody/bodysystemcuda.cu

@@ -0,0 +1,286 @@
+/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <helper_cuda.h>
+#include <math.h>
+
+#if defined(__APPLE__) || defined(MACOSX)
+#pragma clang diagnostic ignored "-Wdeprecated-declarations"
+#include <GLUT/glut.h>
+#else
+#include <GL/freeglut.h>
+#endif
+
+// CUDA standard includes
+#include <cuda_runtime.h>
+#include <cuda_gl_interop.h>
+
+#include <cooperative_groups.h>
+
+namespace cg = cooperative_groups;
+
+#include "bodysystem.h"
+
+__constant__ float softeningSquared;
+__constant__ double softeningSquared_fp64;
+
+cudaError_t setSofteningSquared(float softeningSq) {
+  return cudaMemcpyToSymbol(softeningSquared, &softeningSq, sizeof(float), 0,
+                            cudaMemcpyHostToDevice);
+}
+
+cudaError_t setSofteningSquared(double softeningSq) {
+  return cudaMemcpyToSymbol(softeningSquared_fp64, &softeningSq, sizeof(double),
+                            0, cudaMemcpyHostToDevice);
+}
+
+template <class T>
+struct SharedMemory {
+  __device__ inline operator T *() {
+    extern __shared__ int __smem[];
+    return (T *)__smem;
+  }
+
+  __device__ inline operator const T *() const {
+    extern __shared__ int __smem[];
+    return (T *)__smem;
+  }
+};
+
+template <typename T>
+__device__ T rsqrt_T(T x) {
+  return rsqrt(x);
+}
+
+template <>
+__device__ float rsqrt_T<float>(float x) {
+  return rsqrtf(x);
+}
+
+template <>
+__device__ double rsqrt_T<double>(double x) {
+  return rsqrt(x);
+}
+
+// Macros to simplify shared memory addressing
+#define SX(i) sharedPos[i + blockDim.x * threadIdx.y]
+// This macro is only used when multithreadBodies is true (below)
+#define SX_SUM(i, j) sharedPos[i + blockDim.x * j]
+
+template <typename T>
+__device__ T getSofteningSquared() {
+  return softeningSquared;
+}
+template <>
+__device__ double getSofteningSquared<double>() {
+  return softeningSquared_fp64;
+}
+
+template <typename T>
+struct DeviceData {
+  T *dPos[2];  // mapped host pointers
+  T *dVel;
+  cudaEvent_t event;
+  unsigned int offset;
+  unsigned int numBodies;
+};
+
+template <typename T>
+__device__ typename vec3<T>::Type bodyBodyInteraction(
+    typename vec3<T>::Type ai, typename vec4<T>::Type bi,
+    typename vec4<T>::Type bj) {
+  typename vec3<T>::Type r;
+
+  // r_ij  [3 FLOPS]
+  r.x = bj.x - bi.x;
+  r.y = bj.y - bi.y;
+  r.z = bj.z - bi.z;
+
+  // distSqr = dot(r_ij, r_ij) + EPS^2  [6 FLOPS]
+  T distSqr = r.x * r.x + r.y * r.y + r.z * r.z;
+  distSqr += getSofteningSquared<T>();
+
+  // invDistCube =1/distSqr^(3/2)  [4 FLOPS (2 mul, 1 sqrt, 1 inv)]
+  T invDist = rsqrt_T(distSqr);
+  T invDistCube = invDist * invDist * invDist;
+
+  // s = m_j * invDistCube [1 FLOP]
+  T s = bj.w * invDistCube;
+
+  // a_i =  a_i + s * r_ij [6 FLOPS]
+  ai.x += r.x * s;
+  ai.y += r.y * s;
+  ai.z += r.z * s;
+
+  return ai;
+}
+
+template <typename T>
+__device__ typename vec3<T>::Type computeBodyAccel(
+    typename vec4<T>::Type bodyPos, typename vec4<T>::Type *positions,
+    int numTiles, cg::thread_block cta) {
+  typename vec4<T>::Type *sharedPos = SharedMemory<typename vec4<T>::Type>();
+
+  typename vec3<T>::Type acc = {0.0f, 0.0f, 0.0f};
+
+  for (int tile = 0; tile < numTiles; tile++) {
+    sharedPos[threadIdx.x] = positions[tile * blockDim.x + threadIdx.x];
+
+    cg::sync(cta);
+
+// This is the "tile_calculation" from the GPUG3 article.
+#pragma unroll 128
+
+    for (unsigned int counter = 0; counter < blockDim.x; counter++) {
+      acc = bodyBodyInteraction<T>(acc, bodyPos, sharedPos[counter]);
+    }
+
+    cg::sync(cta);
+  }
+
+  return acc;
+}
+
+template <typename T>
+__global__ void integrateBodies(typename vec4<T>::Type *__restrict__ newPos,
+                                typename vec4<T>::Type *__restrict__ oldPos,
+                                typename vec4<T>::Type *vel,
+                                unsigned int deviceOffset,
+                                unsigned int deviceNumBodies, float deltaTime,
+                                float damping, int numTiles) {
+  // Handle to thread block group
+  cg::thread_block cta = cg::this_thread_block();
+  int index = blockIdx.x * blockDim.x + threadIdx.x;
+
+  if (index >= deviceNumBodies) {
+    return;
+  }
+
+  typename vec4<T>::Type position = oldPos[deviceOffset + index];
+
+  typename vec3<T>::Type accel =
+      computeBodyAccel<T>(position, oldPos, numTiles, cta);
+
+  // acceleration = force / mass;
+  // new velocity = old velocity + acceleration * deltaTime
+  // note we factor out the body's mass from the equation, here and in
+  // bodyBodyInteraction
+  // (because they cancel out).  Thus here force == acceleration
+  typename vec4<T>::Type velocity = vel[deviceOffset + index];
+
+  velocity.x += accel.x * deltaTime;
+  velocity.y += accel.y * deltaTime;
+  velocity.z += accel.z * deltaTime;
+
+  velocity.x *= damping;
+  velocity.y *= damping;
+  velocity.z *= damping;
+
+  // new position = old position + velocity * deltaTime
+  position.x += velocity.x * deltaTime;
+  position.y += velocity.y * deltaTime;
+  position.z += velocity.z * deltaTime;
+
+  // store new position and velocity
+  newPos[deviceOffset + index] = position;
+  vel[deviceOffset + index] = velocity;
+}
+
+template <typename T>
+void integrateNbodySystem(DeviceData<T> *deviceData,
+                          cudaGraphicsResource **pgres,
+                          unsigned int currentRead, float deltaTime,
+                          float damping, unsigned int numBodies,
+                          unsigned int numDevices, int blockSize,
+                          bool bUsePBO) {
+  if (bUsePBO) {
+    checkCudaErrors(cudaGraphicsResourceSetMapFlags(
+        pgres[currentRead], cudaGraphicsMapFlagsReadOnly));
+    checkCudaErrors(cudaGraphicsResourceSetMapFlags(
+        pgres[1 - currentRead], cudaGraphicsMapFlagsWriteDiscard));
+    checkCudaErrors(cudaGraphicsMapResources(2, pgres, 0));
+    size_t bytes;
+    checkCudaErrors(cudaGraphicsResourceGetMappedPointer(
+        (void **)&(deviceData[0].dPos[currentRead]), &bytes,
+        pgres[currentRead]));
+    checkCudaErrors(cudaGraphicsResourceGetMappedPointer(
+        (void **)&(deviceData[0].dPos[1 - currentRead]), &bytes,
+        pgres[1 - currentRead]));
+  }
+
+  for (unsigned int dev = 0; dev != numDevices; dev++) {
+    if (numDevices > 1) {
+      cudaSetDevice(dev);
+    }
+
+    int numBlocks = (deviceData[dev].numBodies + blockSize - 1) / blockSize;
+    int numTiles = (numBodies + blockSize - 1) / blockSize;
+    int sharedMemSize = blockSize * 4 * sizeof(T);  // 4 floats for pos
+
+    integrateBodies<T><<<numBlocks, blockSize, sharedMemSize>>>(
+        (typename vec4<T>::Type *)deviceData[dev].dPos[1 - currentRead],
+        (typename vec4<T>::Type *)deviceData[dev].dPos[currentRead],
+        (typename vec4<T>::Type *)deviceData[dev].dVel, deviceData[dev].offset,
+        deviceData[dev].numBodies, deltaTime, damping, numTiles);
+
+    if (numDevices > 1) {
+      checkCudaErrors(cudaEventRecord(deviceData[dev].event));
+      // MJH: Hack on older driver versions to force kernel launches to flush!
+      cudaStreamQuery(0);
+    }
+
+    // check if kernel invocation generated an error
+    getLastCudaError("Kernel execution failed");
+  }
+
+  if (numDevices > 1) {
+    for (unsigned int dev = 0; dev < numDevices; dev++) {
+      checkCudaErrors(cudaEventSynchronize(deviceData[dev].event));
+    }
+  }
+
+  if (bUsePBO) {
+    checkCudaErrors(cudaGraphicsUnmapResources(2, pgres, 0));
+  }
+}
+
+// Explicit specializations needed to generate code
+template void integrateNbodySystem<float>(DeviceData<float> *deviceData,
+                                          cudaGraphicsResource **pgres,
+                                          unsigned int currentRead,
+                                          float deltaTime, float damping,
+                                          unsigned int numBodies,
+                                          unsigned int numDevices,
+                                          int blockSize, bool bUsePBO);
+
+template void integrateNbodySystem<double>(DeviceData<double> *deviceData,
+                                           cudaGraphicsResource **pgres,
+                                           unsigned int currentRead,
+                                           float deltaTime, float damping,
+                                           unsigned int numBodies,
+                                           unsigned int numDevices,
+                                           int blockSize, bool bUsePBO);

Samples/5_Domain_Specific/nbody/bodysystemcuda.h

@@ -0,0 +1,102 @@
+/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __BODYSYSTEMCUDA_H__
+#define __BODYSYSTEMCUDA_H__
+
+#include "bodysystem.h"
+
+template <typename T>
+struct DeviceData {
+  T *dPos[2];  // mapped host pointers
+  T *dVel;
+  cudaEvent_t event;
+  unsigned int offset;
+  unsigned int numBodies;
+};
+
+// CUDA BodySystem: runs on the GPU
+template <typename T>
+class BodySystemCUDA : public BodySystem<T> {
+ public:
+  BodySystemCUDA(unsigned int numBodies, unsigned int numDevices,
+                 unsigned int blockSize, bool usePBO, bool useSysMem = false,
+                 bool useP2P = true, int deviceId = 0);
+  virtual ~BodySystemCUDA();
+
+  virtual void loadTipsyFile(const std::string &filename);
+
+  virtual void update(T deltaTime);
+
+  virtual void setSoftening(T softening);
+  virtual void setDamping(T damping);
+
+  virtual T *getArray(BodyArray array);
+  virtual void setArray(BodyArray array, const T *data);
+
+  virtual unsigned int getCurrentReadBuffer() const {
+    return m_pbo[m_currentRead];
+  }
+
+  virtual unsigned int getNumBodies() const { return m_numBodies; }
+
+ protected:  // methods
+  BodySystemCUDA() {}
+
+  virtual void _initialize(int numBodies);
+  virtual void _finalize();
+
+ protected:  // data
+  unsigned int m_numBodies;
+  unsigned int m_numDevices;
+  bool m_bInitialized;
+  int m_devID;
+
+  // Host data
+  T *m_hPos[2];
+  T *m_hVel;
+
+  DeviceData<T> *m_deviceData;
+
+  bool m_bUsePBO;
+  bool m_bUseSysMem;
+  bool m_bUseP2P;
+  unsigned int m_SMVersion;
+
+  T m_damping;
+
+  unsigned int m_pbo[2];
+  cudaGraphicsResource *m_pGRes[2];
+  unsigned int m_currentRead;
+  unsigned int m_currentWrite;
+
+  unsigned int m_blockSize;
+};
+
+#include "bodysystemcuda_impl.h"
+
+#endif  // __BODYSYSTEMCUDA_H__

Samples/5_Domain_Specific/nbody/bodysystemcuda_impl.h

@@ -0,0 +1,414 @@
+/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <helper_cuda.h>
+
+#include <assert.h>
+#include <math.h>
+#include <memory.h>
+#include <cstdio>
+#include <cstdlib>
+#include <vector>
+#include <algorithm>
+
+#include <cuda_gl_interop.h>
+
+template <typename T>
+void integrateNbodySystem(DeviceData<T> *deviceData,
+                          cudaGraphicsResource **pgres,
+                          unsigned int currentRead, float deltaTime,
+                          float damping, unsigned int numBodies,
+                          unsigned int numDevices, int blockSize, bool bUsePBO);
+
+cudaError_t setSofteningSquared(float softeningSq);
+cudaError_t setSofteningSquared(double softeningSq);
+
+template <typename T>
+BodySystemCUDA<T>::BodySystemCUDA(unsigned int numBodies,
+                                  unsigned int numDevices,
+                                  unsigned int blockSize, bool usePBO,
+                                  bool useSysMem, bool useP2P, int deviceId)
+    : m_numBodies(numBodies),
+      m_numDevices(numDevices),
+      m_bInitialized(false),
+      m_bUsePBO(usePBO),
+      m_bUseSysMem(useSysMem),
+      m_bUseP2P(useP2P),
+      m_currentRead(0),
+      m_currentWrite(1),
+      m_blockSize(blockSize),
+      m_devID(deviceId) {
+  m_hPos[0] = m_hPos[1] = 0;
+  m_hVel = 0;
+
+  m_deviceData = 0;
+
+  _initialize(numBodies);
+  setSoftening(0.00125f);
+  setDamping(0.995f);
+}
+
+template <typename T>
+BodySystemCUDA<T>::~BodySystemCUDA() {
+  _finalize();
+  m_numBodies = 0;
+}
+
+template <typename T>
+void BodySystemCUDA<T>::_initialize(int numBodies) {
+  assert(!m_bInitialized);
+
+  m_numBodies = numBodies;
+
+  unsigned int memSize = sizeof(T) * 4 * numBodies;
+
+  m_deviceData = new DeviceData<T>[m_numDevices];
+
+  // divide up the workload amongst Devices
+  float *weights = new float[m_numDevices];
+  int *numSms = new int[m_numDevices];
+  float total = 0;
+
+  for (unsigned int i = 0; i < m_numDevices; i++) {
+    cudaDeviceProp props;
+    checkCudaErrors(cudaGetDeviceProperties(&props, i));
+
+    // Choose the weight based on the Compute Capability
+    // We estimate that a CC2.0 SM is about 4.0x faster than a CC 1.x SM for
+    // this application (since a 15-SM GF100 is about 2X faster than a 30-SM
+    // GT200).
+    numSms[i] = props.multiProcessorCount;
+    weights[i] = numSms[i] * (props.major >= 2 ? 4.f : 1.f);
+    total += weights[i];
+  }
+
+  unsigned int offset = 0;
+  unsigned int remaining = m_numBodies;
+
+  for (unsigned int i = 0; i < m_numDevices; i++) {
+    unsigned int count = (int)((weights[i] / total) * m_numBodies);
+    // Rounding up to numSms[i]*256 leads to better GPU utilization _per_ GPU
+    // but when using multiple devices, it will lead to the last GPUs not having
+    // any work at all
+    // which means worse overall performance
+    // unsigned int round = numSms[i] * 256;
+    unsigned int round = 256;
+
+    count = round * ((count + round - 1) / round);
+    if (count > remaining) {
+      count = remaining;
+    }
+
+    remaining -= count;
+    m_deviceData[i].offset = offset;
+    m_deviceData[i].numBodies = count;
+    offset += count;
+
+    if ((i == m_numDevices - 1) && (offset < m_numBodies - 1)) {
+      m_deviceData[i].numBodies += m_numBodies - offset;
+    }
+  }
+
+  delete[] weights;
+  delete[] numSms;
+
+  if (m_bUseSysMem) {
+    checkCudaErrors(cudaHostAlloc((void **)&m_hPos[0], memSize,
+                                  cudaHostAllocMapped | cudaHostAllocPortable));
+    checkCudaErrors(cudaHostAlloc((void **)&m_hPos[1], memSize,
+                                  cudaHostAllocMapped | cudaHostAllocPortable));
+    checkCudaErrors(cudaHostAlloc((void **)&m_hVel, memSize,
+                                  cudaHostAllocMapped | cudaHostAllocPortable));
+
+    memset(m_hPos[0], 0, memSize);
+    memset(m_hPos[1], 0, memSize);
+    memset(m_hVel, 0, memSize);
+
+    for (unsigned int i = 0; i < m_numDevices; i++) {
+      if (m_numDevices > 1) {
+        checkCudaErrors(cudaSetDevice(i));
+      }
+
+      checkCudaErrors(cudaEventCreate(&m_deviceData[i].event));
+      checkCudaErrors(cudaHostGetDevicePointer(
+          (void **)&m_deviceData[i].dPos[0], (void *)m_hPos[0], 0));
+      checkCudaErrors(cudaHostGetDevicePointer(
+          (void **)&m_deviceData[i].dPos[1], (void *)m_hPos[1], 0));
+      checkCudaErrors(cudaHostGetDevicePointer((void **)&m_deviceData[i].dVel,
+                                               (void *)m_hVel, 0));
+    }
+  } else {
+    m_hPos[0] = new T[m_numBodies * 4];
+    m_hVel = new T[m_numBodies * 4];
+
+    memset(m_hPos[0], 0, memSize);
+    memset(m_hVel, 0, memSize);
+
+    checkCudaErrors(cudaSetDevice(m_devID));
+    checkCudaErrors(cudaEventCreate(&m_deviceData[0].event));
+
+    if (m_bUsePBO) {
+      // create the position pixel buffer objects for rendering
+      // we will actually compute directly from this memory in CUDA too
+      glGenBuffers(2, (GLuint *)m_pbo);
+
+      for (int i = 0; i < 2; ++i) {
+        glBindBuffer(GL_ARRAY_BUFFER, m_pbo[i]);
+        glBufferData(GL_ARRAY_BUFFER, memSize, m_hPos[0], GL_DYNAMIC_DRAW);
+
+        int size = 0;
+        glGetBufferParameteriv(GL_ARRAY_BUFFER, GL_BUFFER_SIZE, (GLint *)&size);
+
+        if ((unsigned)size != memSize) {
+          fprintf(stderr, "WARNING: Pixel Buffer Object allocation failed!n");
+        }
+
+        glBindBuffer(GL_ARRAY_BUFFER, 0);
+        checkCudaErrors(cudaGraphicsGLRegisterBuffer(&m_pGRes[i], m_pbo[i],
+                                                     cudaGraphicsMapFlagsNone));
+      }
+    } else {
+      checkCudaErrors(cudaMalloc((void **)&m_deviceData[0].dPos[0], memSize));
+      checkCudaErrors(cudaMalloc((void **)&m_deviceData[0].dPos[1], memSize));
+    }
+
+    checkCudaErrors(cudaMalloc((void **)&m_deviceData[0].dVel, memSize));
+
+    // At this point we already know P2P is supported
+    if (m_bUseP2P) {
+      for (unsigned int i = 1; i < m_numDevices; i++) {
+        int access = 0;
+        cudaError_t error;
+
+        // Enable access for gpu_i to memory owned by gpu0
+        checkCudaErrors(cudaSetDevice(i));
+        if ((error = cudaDeviceEnablePeerAccess(0, 0)) !=
+            cudaErrorPeerAccessAlreadyEnabled) {
+          checkCudaErrors(error);
+        } else {
+          // We might have already enabled P2P, so catch this and reset error
+          // code...
+          cudaGetLastError();
+        }
+
+        checkCudaErrors(cudaEventCreate(&m_deviceData[i].event));
+
+        // Point all GPUs to the memory allocated on gpu0
+        m_deviceData[i].dPos[0] = m_deviceData[0].dPos[0];
+        m_deviceData[i].dPos[1] = m_deviceData[0].dPos[1];
+        m_deviceData[i].dVel = m_deviceData[0].dVel;
+      }
+    }
+  }
+
+  m_bInitialized = true;
+}
+
+template <typename T>
+void BodySystemCUDA<T>::_finalize() {
+  assert(m_bInitialized);
+
+  if (m_bUseSysMem) {
+    checkCudaErrors(cudaFreeHost(m_hPos[0]));
+    checkCudaErrors(cudaFreeHost(m_hPos[1]));
+    checkCudaErrors(cudaFreeHost(m_hVel));
+
+    for (unsigned int i = 0; i < m_numDevices; i++) {
+      cudaEventDestroy(m_deviceData[i].event);
+    }
+  } else {
+    delete[] m_hPos[0];
+    delete[] m_hPos[1];
+    delete[] m_hVel;
+
+    checkCudaErrors(cudaFree((void **)m_deviceData[0].dVel));
+
+    if (m_bUsePBO) {
+      checkCudaErrors(cudaGraphicsUnregisterResource(m_pGRes[0]));
+      checkCudaErrors(cudaGraphicsUnregisterResource(m_pGRes[1]));
+      glDeleteBuffers(2, (const GLuint *)m_pbo);
+    } else {
+      checkCudaErrors(cudaFree((void **)m_deviceData[0].dPos[0]));
+      checkCudaErrors(cudaFree((void **)m_deviceData[0].dPos[1]));
+
+      checkCudaErrors(cudaEventDestroy(m_deviceData[0].event));
+
+      if (m_bUseP2P) {
+        for (unsigned int i = 1; i < m_numDevices; i++) {
+          checkCudaErrors(cudaEventDestroy(m_deviceData[i].event));
+        }
+      }
+    }
+  }
+
+  delete[] m_deviceData;
+
+  m_bInitialized = false;
+}
+
+template <typename T>
+void BodySystemCUDA<T>::loadTipsyFile(const std::string &filename) {
+  if (m_bInitialized) _finalize();
+
+  std::vector<typename vec4<T>::Type> positions;
+  std::vector<typename vec4<T>::Type> velocities;
+  std::vector<int> ids;
+
+  int nBodies = 0;
+  int nFirst = 0, nSecond = 0, nThird = 0;
+
+  read_tipsy_file(positions, velocities, ids, filename, nBodies, nFirst,
+                  nSecond, nThird);
+
+  _initialize(nBodies);
+
+  setArray(BODYSYSTEM_POSITION, (T *)&positions[0]);
+  setArray(BODYSYSTEM_VELOCITY, (T *)&velocities[0]);
+}
+
+template <typename T>
+void BodySystemCUDA<T>::setSoftening(T softening) {
+  T softeningSq = softening * softening;
+
+  for (unsigned int i = 0; i < m_numDevices; i++) {
+    if (m_numDevices > 1) {
+      checkCudaErrors(cudaSetDevice(i));
+    }
+
+    checkCudaErrors(setSofteningSquared(softeningSq));
+  }
+}
+
+template <typename T>
+void BodySystemCUDA<T>::setDamping(T damping) {
+  m_damping = damping;
+}
+
+template <typename T>
+void BodySystemCUDA<T>::update(T deltaTime) {
+  assert(m_bInitialized);
+
+  integrateNbodySystem<T>(m_deviceData, m_pGRes, m_currentRead,
+                          (float)deltaTime, (float)m_damping, m_numBodies,
+                          m_numDevices, m_blockSize, m_bUsePBO);
+
+  std::swap(m_currentRead, m_currentWrite);
+}
+
+template <typename T>
+T *BodySystemCUDA<T>::getArray(BodyArray array) {
+  assert(m_bInitialized);
+
+  T *hdata = 0;
+  T *ddata = 0;
+
+  cudaGraphicsResource *pgres = NULL;
+
+  int currentReadHost = m_bUseSysMem ? m_currentRead : 0;
+
+  switch (array) {
+    default:
+    case BODYSYSTEM_POSITION:
+      hdata = m_hPos[currentReadHost];
+      ddata = m_deviceData[0].dPos[m_currentRead];
+
+      if (m_bUsePBO) {
+        pgres = m_pGRes[m_currentRead];
+      }
+
+      break;
+
+    case BODYSYSTEM_VELOCITY:
+      hdata = m_hVel;
+      ddata = m_deviceData[0].dVel;
+      break;
+  }
+
+  if (!m_bUseSysMem) {
+    if (pgres) {
+      checkCudaErrors(
+          cudaGraphicsResourceSetMapFlags(pgres, cudaGraphicsMapFlagsReadOnly));
+      checkCudaErrors(cudaGraphicsMapResources(1, &pgres, 0));
+      size_t bytes;
+      checkCudaErrors(
+          cudaGraphicsResourceGetMappedPointer((void **)&ddata, &bytes, pgres));
+    }
+
+    checkCudaErrors(cudaMemcpy(hdata, ddata, m_numBodies * 4 * sizeof(T),
+                               cudaMemcpyDeviceToHost));
+
+    if (pgres) {
+      checkCudaErrors(cudaGraphicsUnmapResources(1, &pgres, 0));
+    }
+  }
+
+  return hdata;
+}
+
+template <typename T>
+void BodySystemCUDA<T>::setArray(BodyArray array, const T *data) {
+  assert(m_bInitialized);
+
+  m_currentRead = 0;
+  m_currentWrite = 1;
+
+  switch (array) {
+    default:
+    case BODYSYSTEM_POSITION: {
+      if (m_bUsePBO) {
+        glBindBuffer(GL_ARRAY_BUFFER, m_pbo[m_currentRead]);
+        glBufferSubData(GL_ARRAY_BUFFER, 0, 4 * sizeof(T) * m_numBodies, data);
+
+        int size = 0;
+        glGetBufferParameteriv(GL_ARRAY_BUFFER, GL_BUFFER_SIZE, (GLint *)&size);
+
+        if ((unsigned)size != 4 * (sizeof(T) * m_numBodies)) {
+          fprintf(stderr, "WARNING: Pixel Buffer Object download failed!n");
+        }
+
+        glBindBuffer(GL_ARRAY_BUFFER, 0);
+      } else {
+        if (m_bUseSysMem) {
+          memcpy(m_hPos[m_currentRead], data, m_numBodies * 4 * sizeof(T));
+        } else
+          checkCudaErrors(cudaMemcpy(m_deviceData[0].dPos[m_currentRead], data,
+                                     m_numBodies * 4 * sizeof(T),
+                                     cudaMemcpyHostToDevice));
+      }
+    } break;
+
+    case BODYSYSTEM_VELOCITY:
+      if (m_bUseSysMem) {
+        memcpy(m_hVel, data, m_numBodies * 4 * sizeof(T));
+      } else
+        checkCudaErrors(cudaMemcpy(m_deviceData[0].dVel, data,
+                                   m_numBodies * 4 * sizeof(T),
+                                   cudaMemcpyHostToDevice));
+
+      break;
+  }
+}

Samples/5_Domain_Specific/nbody/findgllib.mk

@@ -0,0 +1,145 @@
+################################################################################
+# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#  * Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#  * Neither the name of NVIDIA CORPORATION nor the names of its
+#    contributors may be used to endorse or promote products derived
+#    from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+################################################################################
+#
+#  findgllib.mk is used to find the necessary GL Libraries for specific distributions
+#               this is supported on Mac OSX and Linux Platforms
+#
+################################################################################
+
+# Determine OS platform and unix distribution
+ifeq ("$(TARGET_OS)","linux")
+   # first search lsb_release
+   DISTRO  := $(shell lsb_release -i -s 2>/dev/null | tr "[:upper:]" "[:lower:]")
+   ifeq ("$(DISTRO)","")
+     # second search and parse /etc/issue
+     DISTRO := $(shell awk '{print $$1}' /etc/issue | tr -d "[:space:]" | sed -e "/^$$/d" | tr "[:upper:]" "[:lower:]")
+     # ensure data from /etc/issue is valid
+     ifeq (,$(filter $(DISTRO),ubuntu fedora red rhel centos suse))
+       DISTRO := 
+     endif
+     ifeq ("$(DISTRO)","")
+       # third, we can search in /etc/os-release or /etc/{distro}-release
+       DISTRO := $(shell awk '/ID/' /etc/*-release | sed 's/ID=//' | grep -v "VERSION" | grep -v "ID" | grep -v "DISTRIB")
+     endif
+   endif
+endif
+
+ifeq ("$(TARGET_OS)","linux")
+    # $(info) >> findgllib.mk -> LINUX path <<<)
+    # Each set of Linux Distros have different paths for where to find their OpenGL libraries reside
+    UBUNTU = $(shell echo $(DISTRO) | grep -i ubuntu      >/dev/null 2>&1; echo $$?)
+    FEDORA = $(shell echo $(DISTRO) | grep -i fedora      >/dev/null 2>&1; echo $$?)
+    RHEL   = $(shell echo $(DISTRO) | grep -i 'red\|rhel' >/dev/null 2>&1; echo $$?)
+    CENTOS = $(shell echo $(DISTRO) | grep -i centos      >/dev/null 2>&1; echo $$?)
+    SUSE   = $(shell echo $(DISTRO) | grep -i 'suse\|sles' >/dev/null 2>&1; echo $$?)
+    ifeq ("$(UBUNTU)","0")
+      ifeq ($(HOST_ARCH)-$(TARGET_ARCH),x86_64-armv7l)
+        GLPATH := /usr/arm-linux-gnueabihf/lib
+        GLLINK := -L/usr/arm-linux-gnueabihf/lib
+        ifneq ($(TARGET_FS),) 
+          GLPATH += $(TARGET_FS)/usr/lib/arm-linux-gnueabihf
+          GLLINK += -L$(TARGET_FS)/usr/lib/arm-linux-gnueabihf
+        endif 
+      else ifeq ($(HOST_ARCH)-$(TARGET_ARCH),x86_64-aarch64)
+        GLPATH := /usr/aarch64-linux-gnu/lib
+        GLLINK := -L/usr/aarch64-linux-gnu/lib
+        ifneq ($(TARGET_FS),) 
+          GLPATH += $(TARGET_FS)/usr/lib
+          GLPATH += $(TARGET_FS)/usr/lib/aarch64-linux-gnu
+          GLLINK += -L$(TARGET_FS)/usr/lib/aarch64-linux-gnu
+        endif 
+      else ifeq ($(HOST_ARCH)-$(TARGET_ARCH),x86_64-ppc64le)
+        GLPATH := /usr/powerpc64le-linux-gnu/lib
+        GLLINK := -L/usr/powerpc64le-linux-gnu/lib
+      else
+        UBUNTU_PKG_NAME = $(shell which dpkg >/dev/null 2>&1 && dpkg -l 'nvidia-*' | grep '^ii' | awk '{print $$2}' | head -1)
+        ifneq ("$(UBUNTU_PKG_NAME)","")
+          GLPATH    ?= /usr/lib/$(UBUNTU_PKG_NAME)
+          GLLINK    ?= -L/usr/lib/$(UBUNTU_PKG_NAME)
+        endif
+
+        DFLT_PATH ?= /usr/lib
+      endif
+    endif
+    ifeq ("$(SUSE)","0")
+      GLPATH    ?= /usr/X11R6/lib64
+      GLLINK    ?= -L/usr/X11R6/lib64
+      DFLT_PATH ?= /usr/lib64
+    endif
+    ifeq ("$(FEDORA)","0")
+      GLPATH    ?= /usr/lib64/nvidia
+      GLLINK    ?= -L/usr/lib64/nvidia
+      DFLT_PATH ?= /usr/lib64
+    endif
+    ifeq ("$(RHEL)","0")
+      GLPATH    ?= /usr/lib64/nvidia
+      GLLINK    ?= -L/usr/lib64/nvidia
+      DFLT_PATH ?= /usr/lib64
+    endif
+    ifeq ("$(CENTOS)","0")
+      GLPATH    ?= /usr/lib64/nvidia
+      GLLINK    ?= -L/usr/lib64/nvidia
+      DFLT_PATH ?= /usr/lib64
+    endif
+  
+  # find libGL, libGLU 
+  GLLIB  := $(shell find -L $(GLPATH) $(DFLT_PATH) -name libGL.so  -print 2>/dev/null)
+  GLULIB := $(shell find -L $(GLPATH) $(DFLT_PATH) -name libGLU.so -print 2>/dev/null)
+
+  ifeq ("$(GLLIB)","")
+      $(info >>> WARNING - libGL.so not found, refer to CUDA Getting Started Guide for how to find and install them. <<<)
+      SAMPLE_ENABLED := 0
+  endif
+  ifeq ("$(GLULIB)","")
+      $(info >>> WARNING - libGLU.so not found, refer to CUDA Getting Started Guide for how to find and install them. <<<)
+      SAMPLE_ENABLED := 0
+  endif
+
+  HEADER_SEARCH_PATH ?= $(TARGET_FS)/usr/include
+  ifeq ($(HOST_ARCH)-$(TARGET_ARCH)-$(TARGET_OS),x86_64-armv7l-linux)
+      HEADER_SEARCH_PATH += /usr/arm-linux-gnueabihf/include
+  else ifeq ($(HOST_ARCH)-$(TARGET_ARCH)-$(TARGET_OS),x86_64-aarch64-linux)
+      HEADER_SEARCH_PATH += /usr/aarch64-linux-gnu/include
+  endif
+
+  GLHEADER  := $(shell find -L $(HEADER_SEARCH_PATH) -name gl.h -print 2>/dev/null)
+  GLUHEADER := $(shell find -L $(HEADER_SEARCH_PATH) -name glu.h -print 2>/dev/null)
+
+  ifeq ("$(GLHEADER)","")
+      $(info >>> WARNING - gl.h not found, refer to CUDA Getting Started Guide for how to find and install them. <<<)
+      SAMPLE_ENABLED := 0
+  endif
+  ifeq ("$(GLUHEADER)","")
+      $(info >>> WARNING - glu.h not found, refer to CUDA Getting Started Guide for how to find and install them. <<<)
+      SAMPLE_ENABLED := 0
+  endif
+else
+    # This would be the Mac OS X path if we had to do anything special
+endif
+

Samples/5_Domain_Specific/nbody/nbody_vs2017.sln

@@ -0,0 +1,20 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio 2017
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "nbody", "nbody_vs2017.vcxproj", "{997E0757-EA74-4A4E-A0FC-47D8C8831A15}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|x64 = Debug|x64
+		Release|x64 = Release|x64
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Debug|x64.ActiveCfg = Debug|x64
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Debug|x64.Build.0 = Debug|x64
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Release|x64.ActiveCfg = Release|x64
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Release|x64.Build.0 = Release|x64
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+EndGlobal

Samples/5_Domain_Specific/nbody/nbody_vs2017.vcxproj

@@ -0,0 +1,130 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <CUDAPropsPath Condition="'$(CUDAPropsPath)'==''">$(VCTargetsPath)\BuildCustomizations</CUDAPropsPath>
+  </PropertyGroup>
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Debug|x64">
+      <Configuration>Debug</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|x64">
+      <Configuration>Release</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>{997E0757-EA74-4A4E-A0FC-47D8C8831A15}</ProjectGuid>
+    <RootNamespace>nbody_vs2017</RootNamespace>
+    <ProjectName>nbody</ProjectName>
+    <CudaToolkitCustomDir />
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(WindowsTargetPlatformVersion)'==''">
+    <LatestTargetPlatformVersion>$([Microsoft.Build.Utilities.ToolLocationHelper]::GetLatestSDKTargetPlatformVersion('Windows', '10.0'))</LatestTargetPlatformVersion>
+    <WindowsTargetPlatformVersion Condition="'$(WindowsTargetPlatformVersion)' == ''">$(LatestTargetPlatformVersion)</WindowsTargetPlatformVersion>
+    <TargetPlatformVersion>$(WindowsTargetPlatformVersion)</TargetPlatformVersion>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <PropertyGroup>
+    <ConfigurationType>Application</ConfigurationType>
+    <CharacterSet>MultiByte</CharacterSet>
+    <PlatformToolset>v141</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)'=='Debug'">
+    <UseDebugLibraries>true</UseDebugLibraries>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)'=='Release'">
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <ImportGroup Label="ExtensionSettings">
+    <Import Project="$(CUDAPropsPath)\CUDA 11.6.props" />
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets">
+    <Import Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" />
+  </ImportGroup>
+  <PropertyGroup Label="UserMacros" />
+  <PropertyGroup>
+    <IntDir>$(Platform)/$(Configuration)/</IntDir>
+    <IncludePath>$(IncludePath)</IncludePath>
+    <CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRules />
+    <CodeAnalysisRuleAssemblies />
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Platform)'=='x64'">
+    <OutDir>../../../bin/win64/$(Configuration)/</OutDir>
+  </PropertyGroup>
+  <ItemDefinitionGroup>
+    <ClCompile>
+      <WarningLevel>Level3</WarningLevel>
+      <PreprocessorDefinitions>WIN32;_MBCS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <AdditionalIncludeDirectories>./;$(CudaToolkitDir)/include;../../../Common;</AdditionalIncludeDirectories>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <AdditionalDependencies>cudart_static.lib;freeglut.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalLibraryDirectories>$(CudaToolkitLibDir);../../../Common/lib/$(PlatformName);</AdditionalLibraryDirectories>
+      <OutputFile>$(OutDir)/nbody.exe</OutputFile>
+    </Link>
+    <CudaCompile>
+      <CodeGeneration>compute_35,sm_35;compute_37,sm_37;compute_50,sm_50;compute_52,sm_52;compute_60,sm_60;compute_61,sm_61;compute_70,sm_70;compute_75,sm_75;compute_80,sm_80;compute_86,sm_86;</CodeGeneration>
+      <AdditionalOptions>-Xcompiler "/wd 4819"  --threads 0 </AdditionalOptions>
+      <Include>./;../../../Common</Include>
+      <Defines>WIN32</Defines>
+    </CudaCompile>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)'=='Debug'">
+    <ClCompile>
+      <Optimization>Disabled</Optimization>
+      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <LinkTimeCodeGeneration>Default</LinkTimeCodeGeneration>
+    </Link>
+    <CudaCompile>
+      <Runtime>MTd</Runtime>
+      <TargetMachinePlatform>64</TargetMachinePlatform>
+    </CudaCompile>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)'=='Release'">
+    <ClCompile>
+      <Optimization>MaxSpeed</Optimization>
+      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <GenerateDebugInformation>false</GenerateDebugInformation>
+      <LinkTimeCodeGeneration>UseLinkTimeCodeGeneration</LinkTimeCodeGeneration>
+    </Link>
+    <CudaCompile>
+      <Runtime>MT</Runtime>
+      <TargetMachinePlatform>64</TargetMachinePlatform>
+    </CudaCompile>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Platform)'=='Win32'">
+    <Link>
+      
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Platform)'=='x64'">
+    <Link>
+      <AdditionalDependencies>glew64.lib;%(AdditionalDependencies)</AdditionalDependencies>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemGroup>
+    <CudaCompile Include="bodysystemcuda.cu" />
+    <ClCompile Include="nbody.cpp" />
+    <ClCompile Include="render_particles.cpp" />
+    <ClInclude Include="bodysystem.h" />
+    <ClInclude Include="bodysystemcpu.h" />
+    <ClInclude Include="bodysystemcpu_impl.h" />
+    <ClInclude Include="bodysystemcuda.h" />
+    <ClInclude Include="bodysystemcuda_impl.h" />
+    <ClInclude Include="render_particles.h" />
+    <ClInclude Include="tipsy.h" />
+  </ItemGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <ImportGroup Label="ExtensionTargets">
+    <Import Project="$(CUDAPropsPath)\CUDA 11.6.targets" />
+  </ImportGroup>
+</Project>

Samples/5_Domain_Specific/nbody/nbody_vs2019.sln

@@ -0,0 +1,20 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio 2019
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "nbody", "nbody_vs2019.vcxproj", "{997E0757-EA74-4A4E-A0FC-47D8C8831A15}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|x64 = Debug|x64
+		Release|x64 = Release|x64
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Debug|x64.ActiveCfg = Debug|x64
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Debug|x64.Build.0 = Debug|x64
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Release|x64.ActiveCfg = Release|x64
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Release|x64.Build.0 = Release|x64
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+EndGlobal

Samples/5_Domain_Specific/nbody/nbody_vs2019.vcxproj

@@ -0,0 +1,126 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <CUDAPropsPath Condition="'$(CUDAPropsPath)'==''">$(VCTargetsPath)\BuildCustomizations</CUDAPropsPath>
+  </PropertyGroup>
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Debug|x64">
+      <Configuration>Debug</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|x64">
+      <Configuration>Release</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>{997E0757-EA74-4A4E-A0FC-47D8C8831A15}</ProjectGuid>
+    <RootNamespace>nbody_vs2019</RootNamespace>
+    <ProjectName>nbody</ProjectName>
+    <CudaToolkitCustomDir />
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <PropertyGroup>
+    <ConfigurationType>Application</ConfigurationType>
+    <CharacterSet>MultiByte</CharacterSet>
+    <PlatformToolset>v142</PlatformToolset>
+    <WindowsTargetPlatformVersion>10.0</WindowsTargetPlatformVersion>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)'=='Debug'">
+    <UseDebugLibraries>true</UseDebugLibraries>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)'=='Release'">
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <ImportGroup Label="ExtensionSettings">
+    <Import Project="$(CUDAPropsPath)\CUDA 11.6.props" />
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets">
+    <Import Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" />
+  </ImportGroup>
+  <PropertyGroup Label="UserMacros" />
+  <PropertyGroup>
+    <IntDir>$(Platform)/$(Configuration)/</IntDir>
+    <IncludePath>$(IncludePath)</IncludePath>
+    <CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRules />
+    <CodeAnalysisRuleAssemblies />
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Platform)'=='x64'">
+    <OutDir>../../../bin/win64/$(Configuration)/</OutDir>
+  </PropertyGroup>
+  <ItemDefinitionGroup>
+    <ClCompile>
+      <WarningLevel>Level3</WarningLevel>
+      <PreprocessorDefinitions>WIN32;_MBCS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <AdditionalIncludeDirectories>./;$(CudaToolkitDir)/include;../../../Common;</AdditionalIncludeDirectories>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <AdditionalDependencies>cudart_static.lib;freeglut.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalLibraryDirectories>$(CudaToolkitLibDir);../../../Common/lib/$(PlatformName);</AdditionalLibraryDirectories>
+      <OutputFile>$(OutDir)/nbody.exe</OutputFile>
+    </Link>
+    <CudaCompile>
+      <CodeGeneration>compute_35,sm_35;compute_37,sm_37;compute_50,sm_50;compute_52,sm_52;compute_60,sm_60;compute_61,sm_61;compute_70,sm_70;compute_75,sm_75;compute_80,sm_80;compute_86,sm_86;</CodeGeneration>
+      <AdditionalOptions>-Xcompiler "/wd 4819"  --threads 0 </AdditionalOptions>
+      <Include>./;../../../Common</Include>
+      <Defines>WIN32</Defines>
+    </CudaCompile>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)'=='Debug'">
+    <ClCompile>
+      <Optimization>Disabled</Optimization>
+      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <LinkTimeCodeGeneration>Default</LinkTimeCodeGeneration>
+    </Link>
+    <CudaCompile>
+      <Runtime>MTd</Runtime>
+      <TargetMachinePlatform>64</TargetMachinePlatform>
+    </CudaCompile>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)'=='Release'">
+    <ClCompile>
+      <Optimization>MaxSpeed</Optimization>
+      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <GenerateDebugInformation>false</GenerateDebugInformation>
+      <LinkTimeCodeGeneration>UseLinkTimeCodeGeneration</LinkTimeCodeGeneration>
+    </Link>
+    <CudaCompile>
+      <Runtime>MT</Runtime>
+      <TargetMachinePlatform>64</TargetMachinePlatform>
+    </CudaCompile>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Platform)'=='Win32'">
+    <Link>
+      
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Platform)'=='x64'">
+    <Link>
+      <AdditionalDependencies>glew64.lib;%(AdditionalDependencies)</AdditionalDependencies>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemGroup>
+    <CudaCompile Include="bodysystemcuda.cu" />
+    <ClCompile Include="nbody.cpp" />
+    <ClCompile Include="render_particles.cpp" />
+    <ClInclude Include="bodysystem.h" />
+    <ClInclude Include="bodysystemcpu.h" />
+    <ClInclude Include="bodysystemcpu_impl.h" />
+    <ClInclude Include="bodysystemcuda.h" />
+    <ClInclude Include="bodysystemcuda_impl.h" />
+    <ClInclude Include="render_particles.h" />
+    <ClInclude Include="tipsy.h" />
+  </ItemGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <ImportGroup Label="ExtensionTargets">
+    <Import Project="$(CUDAPropsPath)\CUDA 11.6.targets" />
+  </ImportGroup>
+</Project>

Samples/5_Domain_Specific/nbody/nbody_vs2022.sln

@@ -0,0 +1,20 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio 2022
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "nbody", "nbody_vs2022.vcxproj", "{997E0757-EA74-4A4E-A0FC-47D8C8831A15}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|x64 = Debug|x64
+		Release|x64 = Release|x64
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Debug|x64.ActiveCfg = Debug|x64
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Debug|x64.Build.0 = Debug|x64
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Release|x64.ActiveCfg = Release|x64
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Release|x64.Build.0 = Release|x64
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+EndGlobal

Samples/5_Domain_Specific/nbody/nbody_vs2022.vcxproj

@@ -0,0 +1,126 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <CUDAPropsPath Condition="'$(CUDAPropsPath)'==''">$(VCTargetsPath)\BuildCustomizations</CUDAPropsPath>
+  </PropertyGroup>
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Debug|x64">
+      <Configuration>Debug</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|x64">
+      <Configuration>Release</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>{997E0757-EA74-4A4E-A0FC-47D8C8831A15}</ProjectGuid>
+    <RootNamespace>nbody_vs2022</RootNamespace>
+    <ProjectName>nbody</ProjectName>
+    <CudaToolkitCustomDir />
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <PropertyGroup>
+    <ConfigurationType>Application</ConfigurationType>
+    <CharacterSet>MultiByte</CharacterSet>
+    <PlatformToolset>v143</PlatformToolset>
+    <WindowsTargetPlatformVersion>10.0</WindowsTargetPlatformVersion>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)'=='Debug'">
+    <UseDebugLibraries>true</UseDebugLibraries>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)'=='Release'">
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <ImportGroup Label="ExtensionSettings">
+    <Import Project="$(CUDAPropsPath)\CUDA 11.6.props" />
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets">
+    <Import Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" />
+  </ImportGroup>
+  <PropertyGroup Label="UserMacros" />
+  <PropertyGroup>
+    <IntDir>$(Platform)/$(Configuration)/</IntDir>
+    <IncludePath>$(IncludePath)</IncludePath>
+    <CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRules />
+    <CodeAnalysisRuleAssemblies />
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Platform)'=='x64'">
+    <OutDir>../../../bin/win64/$(Configuration)/</OutDir>
+  </PropertyGroup>
+  <ItemDefinitionGroup>
+    <ClCompile>
+      <WarningLevel>Level3</WarningLevel>
+      <PreprocessorDefinitions>WIN32;_MBCS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <AdditionalIncludeDirectories>./;$(CudaToolkitDir)/include;../../../Common;</AdditionalIncludeDirectories>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <AdditionalDependencies>cudart_static.lib;freeglut.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalLibraryDirectories>$(CudaToolkitLibDir);../../../Common/lib/$(PlatformName);</AdditionalLibraryDirectories>
+      <OutputFile>$(OutDir)/nbody.exe</OutputFile>
+    </Link>
+    <CudaCompile>
+      <CodeGeneration>compute_35,sm_35;compute_37,sm_37;compute_50,sm_50;compute_52,sm_52;compute_60,sm_60;compute_61,sm_61;compute_70,sm_70;compute_75,sm_75;compute_80,sm_80;compute_86,sm_86;</CodeGeneration>
+      <AdditionalOptions>-Xcompiler "/wd 4819"  --threads 0 </AdditionalOptions>
+      <Include>./;../../../Common</Include>
+      <Defines>WIN32</Defines>
+    </CudaCompile>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)'=='Debug'">
+    <ClCompile>
+      <Optimization>Disabled</Optimization>
+      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <LinkTimeCodeGeneration>Default</LinkTimeCodeGeneration>
+    </Link>
+    <CudaCompile>
+      <Runtime>MTd</Runtime>
+      <TargetMachinePlatform>64</TargetMachinePlatform>
+    </CudaCompile>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)'=='Release'">
+    <ClCompile>
+      <Optimization>MaxSpeed</Optimization>
+      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <GenerateDebugInformation>false</GenerateDebugInformation>
+      <LinkTimeCodeGeneration>UseLinkTimeCodeGeneration</LinkTimeCodeGeneration>
+    </Link>
+    <CudaCompile>
+      <Runtime>MT</Runtime>
+      <TargetMachinePlatform>64</TargetMachinePlatform>
+    </CudaCompile>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Platform)'=='Win32'">
+    <Link>
+      
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Platform)'=='x64'">
+    <Link>
+      <AdditionalDependencies>glew64.lib;%(AdditionalDependencies)</AdditionalDependencies>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemGroup>
+    <CudaCompile Include="bodysystemcuda.cu" />
+    <ClCompile Include="nbody.cpp" />
+    <ClCompile Include="render_particles.cpp" />
+    <ClInclude Include="bodysystem.h" />
+    <ClInclude Include="bodysystemcpu.h" />
+    <ClInclude Include="bodysystemcpu_impl.h" />
+    <ClInclude Include="bodysystemcuda.h" />
+    <ClInclude Include="bodysystemcuda_impl.h" />
+    <ClInclude Include="render_particles.h" />
+    <ClInclude Include="tipsy.h" />
+  </ItemGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <ImportGroup Label="ExtensionTargets">
+    <Import Project="$(CUDAPropsPath)\CUDA 11.6.targets" />
+  </ImportGroup>
+</Project>

Samples/5_Domain_Specific/nbody/render_particles.cpp

@@ -0,0 +1,368 @@
+/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "render_particles.h"
+
+#define HELPERGL_EXTERN_GL_FUNC_IMPLEMENTATION
+#include <helper_gl.h>
+
+#include <cuda_runtime.h>
+#include <cuda_gl_interop.h>
+
+#include <helper_cuda.h>
+
+#include <math.h>
+#include <assert.h>
+
+#define GL_POINT_SPRITE_ARB 0x8861
+#define GL_COORD_REPLACE_ARB 0x8862
+#define GL_VERTEX_PROGRAM_POINT_SIZE_NV 0x8642
+
+ParticleRenderer::ParticleRenderer()
+    : m_pos(0),
+      m_numParticles(0),
+      m_pointSize(1.0f),
+      m_spriteSize(2.0f),
+      m_vertexShader(0),
+      m_vertexShaderPoints(0),
+      m_pixelShader(0),
+      m_programPoints(0),
+      m_programSprites(0),
+      m_texture(0),
+      m_pbo(0),
+      m_vboColor(0),
+      m_bFp64Positions(false) {
+  _initGL();
+}
+
+ParticleRenderer::~ParticleRenderer() { m_pos = 0; }
+
+void ParticleRenderer::resetPBO() { glDeleteBuffers(1, (GLuint *)&m_pbo); }
+
+void ParticleRenderer::setPositions(float *pos, int numParticles) {
+  m_pos = pos;
+  m_numParticles = numParticles;
+
+  if (!m_pbo) {
+    glGenBuffers(1, (GLuint *)&m_pbo);
+  }
+
+  glBindBuffer(GL_ARRAY_BUFFER, m_pbo);
+  glBufferData(GL_ARRAY_BUFFER, numParticles * 4 * sizeof(float), pos,
+               GL_STATIC_DRAW);
+  glBindBuffer(GL_ARRAY_BUFFER, 0);
+  SDK_CHECK_ERROR_GL();
+}
+
+void ParticleRenderer::setPositions(double *pos, int numParticles) {
+  m_bFp64Positions = true;
+  m_pos_fp64 = pos;
+  m_numParticles = numParticles;
+
+  if (!m_pbo) {
+    glGenBuffers(1, (GLuint *)&m_pbo);
+  }
+
+  glBindBuffer(GL_ARRAY_BUFFER, m_pbo);
+  glBufferData(GL_ARRAY_BUFFER, numParticles * 4 * sizeof(double), pos,
+               GL_STATIC_DRAW);
+  glBindBuffer(GL_ARRAY_BUFFER, 0);
+  SDK_CHECK_ERROR_GL();
+}
+
+void ParticleRenderer::setColors(float *color, int numParticles) {
+  glBindBuffer(GL_ARRAY_BUFFER, m_vboColor);
+  glBufferData(GL_ARRAY_BUFFER, numParticles * 4 * sizeof(float), color,
+               GL_STATIC_DRAW);
+  glBindBuffer(GL_ARRAY_BUFFER, 0);
+}
+
+void ParticleRenderer::setBaseColor(float color[4]) {
+  for (int i = 0; i < 4; i++) m_baseColor[i] = color[i];
+}
+
+void ParticleRenderer::setPBO(unsigned int pbo, int numParticles, bool fp64) {
+  m_pbo = pbo;
+  m_numParticles = numParticles;
+
+  if (fp64) m_bFp64Positions = true;
+}
+
+void ParticleRenderer::_drawPoints(bool color) {
+  if (!m_pbo) {
+    glBegin(GL_POINTS);
+    {
+      int k = 0;
+
+      for (int i = 0; i < m_numParticles; ++i) {
+        if (m_bFp64Positions)
+          glVertex3dv(&m_pos_fp64[k]);
+        else {
+          glVertex3fv(&m_pos[k]);
+        }
+
+        k += 4;
+      }
+    }
+    glEnd();
+  } else {
+    glEnableClientState(GL_VERTEX_ARRAY);
+
+    glBindBuffer(GL_ARRAY_BUFFER, m_pbo);
+
+    if (m_bFp64Positions)
+      glVertexPointer(4, GL_DOUBLE, 0, 0);
+    else
+      glVertexPointer(4, GL_FLOAT, 0, 0);
+
+    if (color) {
+      glEnableClientState(GL_COLOR_ARRAY);
+      glBindBuffer(GL_ARRAY_BUFFER, m_vboColor);
+      // glActiveTexture(GL_TEXTURE1);
+      // glTexCoordPointer(4, GL_FLOAT, 0, 0);
+      glColorPointer(4, GL_FLOAT, 0, 0);
+    }
+
+    glDrawArrays(GL_POINTS, 0, m_numParticles);
+    glBindBuffer(GL_ARRAY_BUFFER, 0);
+    glDisableClientState(GL_VERTEX_ARRAY);
+    glDisableClientState(GL_COLOR_ARRAY);
+  }
+}
+
+void ParticleRenderer::display(DisplayMode mode /* = PARTICLE_POINTS */) {
+  switch (mode) {
+    case PARTICLE_POINTS:
+      glColor3f(1, 1, 1);
+      glPointSize(m_pointSize);
+      glUseProgram(m_programPoints);
+      _drawPoints();
+      glUseProgram(0);
+      break;
+
+    case PARTICLE_SPRITES:
+    default: {
+      // setup point sprites
+      glEnable(GL_POINT_SPRITE_ARB);
+      glTexEnvi(GL_POINT_SPRITE_ARB, GL_COORD_REPLACE_ARB, GL_TRUE);
+      glEnable(GL_VERTEX_PROGRAM_POINT_SIZE_NV);
+      glPointSize(m_spriteSize);
+      glBlendFunc(GL_SRC_ALPHA, GL_ONE);
+      glEnable(GL_BLEND);
+      glDepthMask(GL_FALSE);
+
+      glUseProgram(m_programSprites);
+      GLuint texLoc = glGetUniformLocation(m_programSprites, "splatTexture");
+      glUniform1i(texLoc, 0);
+
+      glActiveTexture(GL_TEXTURE0);
+      glBindTexture(GL_TEXTURE_2D, m_texture);
+
+      glColor3f(1, 1, 1);
+      glSecondaryColor3fv(m_baseColor);
+
+      _drawPoints();
+
+      glUseProgram(0);
+
+      glDisable(GL_POINT_SPRITE_ARB);
+      glDisable(GL_BLEND);
+      glDepthMask(GL_TRUE);
+    }
+
+    break;
+
+    case PARTICLE_SPRITES_COLOR: {
+      // setup point sprites
+      glEnable(GL_POINT_SPRITE_ARB);
+      glTexEnvi(GL_POINT_SPRITE_ARB, GL_COORD_REPLACE_ARB, GL_TRUE);
+      glEnable(GL_VERTEX_PROGRAM_POINT_SIZE_NV);
+      glPointSize(m_spriteSize);
+      glBlendFunc(GL_SRC_ALPHA, GL_ONE);
+      glEnable(GL_BLEND);
+      glDepthMask(GL_FALSE);
+
+      glUseProgram(m_programSprites);
+      GLuint texLoc = glGetUniformLocation(m_programSprites, "splatTexture");
+      glUniform1i(texLoc, 0);
+
+      glActiveTexture(GL_TEXTURE0);
+      glBindTexture(GL_TEXTURE_2D, m_texture);
+
+      glColor3f(1, 1, 1);
+      glSecondaryColor3fv(m_baseColor);
+
+      _drawPoints(true);
+
+      glUseProgram(0);
+
+      glDisable(GL_POINT_SPRITE_ARB);
+      glDisable(GL_BLEND);
+      glDepthMask(GL_TRUE);
+    }
+
+    break;
+  }
+
+  SDK_CHECK_ERROR_GL();
+}
+
+const char vertexShaderPoints[] = {
+    "void main()                                                            \n"
+    "{                                                                      \n"
+    "    vec4 vert = vec4(gl_Vertex.xyz, 1.0);  			       "
+    " "
+    "              \n"
+    "    gl_Position = gl_ProjectionMatrix * gl_ModelViewMatrix * vert;        "
+    "                   \n"
+    "    gl_FrontColor = gl_Color;                                          \n"
+    "}                                                                      "
+    "\n"};
+
+const char vertexShader[] = {
+    "void main()                                                            \n"
+    "{                                                                      \n"
+    "    float pointSize = 500.0 * gl_Point.size;                           \n"
+    "    vec4 vert = gl_Vertex;						"
+    "						\n"
+    "    vert.w = 1.0;							"
+    "	"
+    "						\n"
+    "    vec3 pos_eye = vec3 (gl_ModelViewMatrix * vert);                   \n"
+    "    gl_PointSize = max(1.0, pointSize / (1.0 - pos_eye.z));            \n"
+    "    gl_TexCoord[0] = gl_MultiTexCoord0;                                \n"
+    //"    gl_TexCoord[1] = gl_MultiTexCoord1; \n"
+    "    gl_Position = gl_ProjectionMatrix * gl_ModelViewMatrix * vert;     \n"
+    "    gl_FrontColor = gl_Color;                                          \n"
+    "    gl_FrontSecondaryColor = gl_SecondaryColor;                        \n"
+    "}                                                                      "
+    "\n"};
+
+const char pixelShader[] = {
+    "uniform sampler2D splatTexture;                                        \n"
+
+    "void main()                                                            \n"
+    "{                                                                      \n"
+    "    vec4 color2 = gl_SecondaryColor;                                   \n"
+    "    vec4 color = (0.6 + 0.4 * gl_Color) * texture2D(splatTexture, "
+    "gl_TexCoord[0].st); \n"
+    "    gl_FragColor =                                                     \n"
+    "         color * color2;\n"  // mix(vec4(0.1, 0.0, 0.0, color.w), color2,
+                                  // color.w);\n"
+    "}                                                                      "
+    "\n"};
+
+void ParticleRenderer::_initGL() {
+  m_vertexShader = glCreateShader(GL_VERTEX_SHADER);
+  m_vertexShaderPoints = glCreateShader(GL_VERTEX_SHADER);
+  m_pixelShader = glCreateShader(GL_FRAGMENT_SHADER);
+
+  const char *v = vertexShader;
+  const char *p = pixelShader;
+  glShaderSource(m_vertexShader, 1, &v, 0);
+  glShaderSource(m_pixelShader, 1, &p, 0);
+  const char *vp = vertexShaderPoints;
+  glShaderSource(m_vertexShaderPoints, 1, &vp, 0);
+
+  glCompileShader(m_vertexShader);
+  glCompileShader(m_vertexShaderPoints);
+  glCompileShader(m_pixelShader);
+
+  m_programSprites = glCreateProgram();
+  glAttachShader(m_programSprites, m_vertexShader);
+  glAttachShader(m_programSprites, m_pixelShader);
+  glLinkProgram(m_programSprites);
+
+  m_programPoints = glCreateProgram();
+  glAttachShader(m_programPoints, m_vertexShaderPoints);
+  glLinkProgram(m_programPoints);
+
+  _createTexture(32);
+
+  glGenBuffers(1, (GLuint *)&m_vboColor);
+  glBindBuffer(GL_ARRAY_BUFFER, m_vboColor);
+  glBufferData(GL_ARRAY_BUFFER, m_numParticles * 4 * sizeof(float), 0,
+               GL_STATIC_DRAW);
+  glBindBuffer(GL_ARRAY_BUFFER, 0);
+}
+
+//------------------------------------------------------------------------------
+// Function           : EvalHermite
+// Description      :
+//------------------------------------------------------------------------------
+/**
+* EvalHermite(float pA, float pB, float vA, float vB, float u)
+* @brief Evaluates Hermite basis functions for the specified coefficients.
+*/
+inline float evalHermite(float pA, float pB, float vA, float vB, float u) {
+  float u2 = (u * u), u3 = u2 * u;
+  float B0 = 2 * u3 - 3 * u2 + 1;
+  float B1 = -2 * u3 + 3 * u2;
+  float B2 = u3 - 2 * u2 + u;
+  float B3 = u3 - u;
+  return (B0 * pA + B1 * pB + B2 * vA + B3 * vB);
+}
+
+unsigned char *createGaussianMap(int N) {
+  float *M = new float[2 * N * N];
+  unsigned char *B = new unsigned char[4 * N * N];
+  float X, Y, Y2, Dist;
+  float Incr = 2.0f / N;
+  int i = 0;
+  int j = 0;
+  Y = -1.0f;
+
+  // float mmax = 0;
+  for (int y = 0; y < N; y++, Y += Incr) {
+    Y2 = Y * Y;
+    X = -1.0f;
+
+    for (int x = 0; x < N; x++, X += Incr, i += 2, j += 4) {
+      Dist = (float)sqrtf(X * X + Y2);
+
+      if (Dist > 1) Dist = 1;
+
+      M[i + 1] = M[i] = evalHermite(1.0f, 0, 0, 0, Dist);
+      B[j + 3] = B[j + 2] = B[j + 1] = B[j] = (unsigned char)(M[i] * 255);
+    }
+  }
+
+  delete[] M;
+  return (B);
+}
+
+void ParticleRenderer::_createTexture(int resolution) {
+  unsigned char *data = createGaussianMap(resolution);
+  glGenTextures(1, (GLuint *)&m_texture);
+  glBindTexture(GL_TEXTURE_2D, m_texture);
+  glTexParameteri(GL_TEXTURE_2D, GL_GENERATE_MIPMAP_SGIS, GL_TRUE);
+  glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER,
+                  GL_LINEAR_MIPMAP_LINEAR);
+  glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
+  glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA8, resolution, resolution, 0, GL_RGBA,
+               GL_UNSIGNED_BYTE, data);
+}

Samples/5_Domain_Specific/nbody/render_particles.h

@@ -0,0 +1,83 @@
+/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __RENDER_PARTICLES__
+#define __RENDER_PARTICLES__
+
+class ParticleRenderer {
+ public:
+  ParticleRenderer();
+  ~ParticleRenderer();
+
+  void setPositions(float *pos, int numParticles);
+  void setPositions(double *pos, int numParticles);
+  void setBaseColor(float color[4]);
+  void setColors(float *color, int numParticles);
+  void setPBO(unsigned int pbo, int numParticles, bool fp64);
+
+  enum DisplayMode {
+    PARTICLE_POINTS,
+    PARTICLE_SPRITES,
+    PARTICLE_SPRITES_COLOR,
+    PARTICLE_NUM_MODES
+  };
+
+  void display(DisplayMode mode = PARTICLE_POINTS);
+
+  void setPointSize(float size) { m_pointSize = size; }
+  void setSpriteSize(float size) { m_spriteSize = size; }
+
+  void resetPBO();
+
+ protected:  // methods
+  void _initGL();
+  void _createTexture(int resolution);
+  void _drawPoints(bool color = false);
+
+ protected:  // data
+  float *m_pos;
+  double *m_pos_fp64;
+  int m_numParticles;
+
+  float m_pointSize;
+  float m_spriteSize;
+
+  unsigned int m_vertexShader;
+  unsigned int m_vertexShaderPoints;
+  unsigned int m_pixelShader;
+  unsigned int m_programPoints;
+  unsigned int m_programSprites;
+  unsigned int m_texture;
+  unsigned int m_pbo;
+  unsigned int m_vboColor;
+
+  float m_baseColor[4];
+
+  bool m_bFp64Positions;
+};
+
+#endif  //__ RENDER_PARTICLES__

Samples/5_Domain_Specific/nbody/tipsy.h

@@ -0,0 +1,172 @@
+#ifndef __TIPSY_H__
+#define __TIPSY_H__
+
+#include <string>
+
+using namespace std;
+
+#define MAXDIM 3
+
+typedef float Real;
+
+struct gas_particle
+{
+    Real mass;
+    Real pos[MAXDIM];
+    Real vel[MAXDIM];
+    Real rho;
+    Real temp;
+    Real hsmooth;
+    Real metals ;
+    Real phi ;
+} ;
+
+//struct gas_particle *gas_particles;
+
+struct dark_particle
+{
+    Real mass;
+    Real pos[MAXDIM];
+    Real vel[MAXDIM];
+    Real eps;
+    int phi ;
+} ;
+
+//struct dark_particle *dark_particles;
+
+struct star_particle
+{
+    Real mass;
+    Real pos[MAXDIM];
+    Real vel[MAXDIM];
+    Real metals ;
+    Real tform ;
+    Real eps;
+    int phi ;
+} ;
+
+//struct star_particle *star_particles;
+
+struct dump
+{
+    double time ;
+    int nbodies ;
+    int ndim ;
+    int nsph ;
+    int ndark ;
+    int nstar ;
+} ;
+
+typedef struct dump header ;
+
+template <typename real4>
+void read_tipsy_file(vector<real4> &bodyPositions,
+                     vector<real4> &bodyVelocities,
+                     vector<int> &bodiesIDs,
+                     const std::string &fileName,
+                     int &NTotal,
+                     int &NFirst,
+                     int &NSecond,
+                     int &NThird)
+{
+    /*
+       Read in our custom version of the tipsy file format written by
+       Jeroen Bedorf.  Most important change is that we store particle id on the
+       location where previously the potential was stored.
+    */
+
+    char fullFileName[256];
+    sprintf(fullFileName, "%s", fileName.c_str());
+
+    cout << "Trying to read file: " << fullFileName << endl;
+
+    ifstream inputFile(fullFileName, ios::in | ios::binary);
+
+    if (!inputFile.is_open())
+    {
+        cout << "Can't open input file \n";
+        exit(EXIT_SUCCESS);
+    }
+
+    dump  h;
+    inputFile.read((char *)&h, sizeof(h));
+
+    int idummy;
+    real4 positions;
+    real4 velocity;
+
+
+    //Read tipsy header
+    NTotal        = h.nbodies;
+    NFirst        = h.ndark;
+    NSecond       = h.nstar;
+    NThird        = h.nsph;
+
+    //Start reading
+    int particleCount = 0;
+
+    dark_particle d;
+    star_particle s;
+
+    for (int i=0; i < NTotal; i++)
+    {
+        if (i < NFirst)
+        {
+            inputFile.read((char *)&d, sizeof(d));
+            velocity.w        = d.eps;
+            positions.w       = d.mass;
+            positions.x       = d.pos[0];
+            positions.y       = d.pos[1];
+            positions.z       = d.pos[2];
+            velocity.x        = d.vel[0];
+            velocity.y        = d.vel[1];
+            velocity.z        = d.vel[2];
+            idummy            = d.phi;
+        }
+        else
+        {
+            inputFile.read((char *)&s, sizeof(s));
+            velocity.w        = s.eps;
+            positions.w       = s.mass;
+            positions.x       = s.pos[0];
+            positions.y       = s.pos[1];
+            positions.z       = s.pos[2];
+            velocity.x        = s.vel[0];
+            velocity.y        = s.vel[1];
+            velocity.z        = s.vel[2];
+            idummy            = s.phi;
+        }
+
+        bodyPositions.push_back(positions);
+        bodyVelocities.push_back(velocity);
+        bodiesIDs.push_back(idummy);
+
+        particleCount++;
+    }//end for
+
+    // round up to a multiple of 256 bodies since our kernel only supports that...
+    int newTotal = NTotal;
+
+    if (NTotal % 256)
+    {
+        newTotal = ((NTotal / 256) + 1) * 256;
+    }
+
+    for (int i = NTotal; i < newTotal; i++)
+    {
+        positions.w = positions.x = positions.y = positions.z = 0;
+        velocity.x = velocity.y = velocity.z = 0;
+        bodyPositions.push_back(positions);
+        bodyVelocities.push_back(velocity);
+        bodiesIDs.push_back(i);
+        NFirst++;
+    }
+
+    NTotal = newTotal;
+
+    inputFile.close();
+
+    cerr << "Read " << NTotal << " bodies" << endl;
+}
+
+#endif //__TIPSY_H__