add and update samples for CUDA 11.6

Samples/6_Performance/README.md

@@ -0,0 +1,12 @@
+# 6. Performance
+
+
+### [alignedTypes](./alignedTypes)
+A simple test, showing huge access speed gap between aligned and misaligned structures. It measures per-element copy throughput for aligned and misaligned structures on big chunks of data.
+
+### [transpose](./transpose)
+This sample demonstrates Matrix Transpose.  Different performance are shown to achieve high performance.
+
+### [UnifiedMemoryPerf](./UnifiedMemoryPerf)
+This sample demonstrates the performance comparision using matrix multiplication kernel of Unified Memory with/without hints and other types of memory like zero copy buffers, pageable, pagelocked memory performing synchronous and Asynchronous transfers on a single GPU.
+

Samples/6_Performance/UnifiedMemoryPerf/.vscode/c_cpp_properties.json

@@ -0,0 +1,18 @@
+{
+    "configurations": [
+        {
+            "name": "Linux",
+            "includePath": [
+                "${workspaceFolder}/**",
+                "${workspaceFolder}/../../../Common"
+            ],
+            "defines": [],
+            "compilerPath": "/usr/local/cuda/bin/nvcc",
+            "cStandard": "gnu17",
+            "cppStandard": "gnu++14",
+            "intelliSenseMode": "linux-gcc-x64",
+            "configurationProvider": "ms-vscode.makefile-tools"
+        }
+    ],
+    "version": 4
+}

Samples/6_Performance/UnifiedMemoryPerf/.vscode/extensions.json

@@ -0,0 +1,7 @@
+{
+    "recommendations": [
+        "nvidia.nsight-vscode-edition",
+        "ms-vscode.cpptools",
+        "ms-vscode.makefile-tools"
+    ]
+}

Samples/6_Performance/UnifiedMemoryPerf/.vscode/launch.json

@@ -0,0 +1,10 @@
+{
+    "configurations": [
+        {
+            "name": "CUDA C++: Launch",
+            "type": "cuda-gdb",
+            "request": "launch",
+            "program": "${workspaceFolder}/UnifiedMemoryPerf"
+        }
+    ]
+}

Samples/6_Performance/UnifiedMemoryPerf/.vscode/tasks.json

@@ -0,0 +1,15 @@
+{
+    "version": "2.0.0",
+    "tasks": [
+        {
+            "label": "sample",
+            "type": "shell",
+            "command": "make dbg=1",
+            "problemMatcher": ["$nvcc"],
+            "group": {
+                "kind": "build",
+                "isDefault": true
+            }
+        }
+    ]
+}

Samples/6_Performance/UnifiedMemoryPerf/Makefile

@@ -0,0 +1,352 @@
+################################################################################
+# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#  * Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#  * Neither the name of NVIDIA CORPORATION nor the names of its
+#    contributors may be used to endorse or promote products derived
+#    from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+################################################################################
+#
+# Makefile project only supported on Mac OS X and Linux Platforms)
+#
+################################################################################
+
+# Location of the CUDA Toolkit
+CUDA_PATH ?= /usr/local/cuda
+
+##############################
+# start deprecated interface #
+##############################
+ifeq ($(x86_64),1)
+    $(info WARNING - x86_64 variable has been deprecated)
+    $(info WARNING - please use TARGET_ARCH=x86_64 instead)
+    TARGET_ARCH ?= x86_64
+endif
+ifeq ($(ARMv7),1)
+    $(info WARNING - ARMv7 variable has been deprecated)
+    $(info WARNING - please use TARGET_ARCH=armv7l instead)
+    TARGET_ARCH ?= armv7l
+endif
+ifeq ($(aarch64),1)
+    $(info WARNING - aarch64 variable has been deprecated)
+    $(info WARNING - please use TARGET_ARCH=aarch64 instead)
+    TARGET_ARCH ?= aarch64
+endif
+ifeq ($(ppc64le),1)
+    $(info WARNING - ppc64le variable has been deprecated)
+    $(info WARNING - please use TARGET_ARCH=ppc64le instead)
+    TARGET_ARCH ?= ppc64le
+endif
+ifneq ($(GCC),)
+    $(info WARNING - GCC variable has been deprecated)
+    $(info WARNING - please use HOST_COMPILER=$(GCC) instead)
+    HOST_COMPILER ?= $(GCC)
+endif
+ifneq ($(abi),)
+    $(error ERROR - abi variable has been removed)
+endif
+############################
+# end deprecated interface #
+############################
+
+# architecture
+HOST_ARCH   := $(shell uname -m)
+TARGET_ARCH ?= $(HOST_ARCH)
+ifneq (,$(filter $(TARGET_ARCH),x86_64 aarch64 sbsa ppc64le armv7l))
+    ifneq ($(TARGET_ARCH),$(HOST_ARCH))
+        ifneq (,$(filter $(TARGET_ARCH),x86_64 aarch64 sbsa ppc64le))
+            TARGET_SIZE := 64
+        else ifneq (,$(filter $(TARGET_ARCH),armv7l))
+            TARGET_SIZE := 32
+        endif
+    else
+        TARGET_SIZE := $(shell getconf LONG_BIT)
+    endif
+else
+    $(error ERROR - unsupported value $(TARGET_ARCH) for TARGET_ARCH!)
+endif
+
+# sbsa and aarch64 systems look similar. Need to differentiate them at host level for now.
+ifeq ($(HOST_ARCH),aarch64)
+    ifeq ($(CUDA_PATH)/targets/sbsa-linux,$(shell ls -1d $(CUDA_PATH)/targets/sbsa-linux 2>/dev/null))
+        HOST_ARCH := sbsa
+        TARGET_ARCH := sbsa
+    endif
+endif
+
+ifneq ($(TARGET_ARCH),$(HOST_ARCH))
+    ifeq (,$(filter $(HOST_ARCH)-$(TARGET_ARCH),aarch64-armv7l x86_64-armv7l x86_64-aarch64 x86_64-sbsa x86_64-ppc64le))
+        $(error ERROR - cross compiling from $(HOST_ARCH) to $(TARGET_ARCH) is not supported!)
+    endif
+endif
+
+# When on native aarch64 system with userspace of 32-bit, change TARGET_ARCH to armv7l
+ifeq ($(HOST_ARCH)-$(TARGET_ARCH)-$(TARGET_SIZE),aarch64-aarch64-32)
+    TARGET_ARCH = armv7l
+endif
+
+# operating system
+HOST_OS   := $(shell uname -s 2>/dev/null | tr "[:upper:]" "[:lower:]")
+TARGET_OS ?= $(HOST_OS)
+ifeq (,$(filter $(TARGET_OS),linux darwin qnx android))
+    $(error ERROR - unsupported value $(TARGET_OS) for TARGET_OS!)
+endif
+
+# host compiler
+ifeq ($(TARGET_OS),darwin)
+    ifeq ($(shell expr `xcodebuild -version | grep -i xcode | awk '{print $$2}' | cut -d'.' -f1` \>= 5),1)
+        HOST_COMPILER ?= clang++
+    endif
+else ifneq ($(TARGET_ARCH),$(HOST_ARCH))
+    ifeq ($(HOST_ARCH)-$(TARGET_ARCH),x86_64-armv7l)
+        ifeq ($(TARGET_OS),linux)
+            HOST_COMPILER ?= arm-linux-gnueabihf-g++
+        else ifeq ($(TARGET_OS),qnx)
+            ifeq ($(QNX_HOST),)
+                $(error ERROR - QNX_HOST must be passed to the QNX host toolchain)
+            endif
+            ifeq ($(QNX_TARGET),)
+                $(error ERROR - QNX_TARGET must be passed to the QNX target toolchain)
+            endif
+            export QNX_HOST
+            export QNX_TARGET
+            HOST_COMPILER ?= $(QNX_HOST)/usr/bin/arm-unknown-nto-qnx6.6.0eabi-g++
+        else ifeq ($(TARGET_OS),android)
+            HOST_COMPILER ?= arm-linux-androideabi-g++
+        endif
+    else ifeq ($(TARGET_ARCH),aarch64)
+        ifeq ($(TARGET_OS), linux)
+            HOST_COMPILER ?= aarch64-linux-gnu-g++
+        else ifeq ($(TARGET_OS),qnx)
+            ifeq ($(QNX_HOST),)
+                $(error ERROR - QNX_HOST must be passed to the QNX host toolchain)
+            endif
+            ifeq ($(QNX_TARGET),)
+                $(error ERROR - QNX_TARGET must be passed to the QNX target toolchain)
+            endif
+            export QNX_HOST
+            export QNX_TARGET
+            HOST_COMPILER ?= $(QNX_HOST)/usr/bin/q++
+        else ifeq ($(TARGET_OS), android)
+            HOST_COMPILER ?= aarch64-linux-android-clang++
+        endif
+    else ifeq ($(TARGET_ARCH),sbsa)
+        HOST_COMPILER ?= aarch64-linux-gnu-g++
+    else ifeq ($(TARGET_ARCH),ppc64le)
+        HOST_COMPILER ?= powerpc64le-linux-gnu-g++
+    endif
+endif
+HOST_COMPILER ?= g++
+NVCC          := $(CUDA_PATH)/bin/nvcc -ccbin $(HOST_COMPILER)
+
+# internal flags
+NVCCFLAGS   := -m${TARGET_SIZE}
+CCFLAGS     :=
+LDFLAGS     :=
+
+# build flags
+ifeq ($(TARGET_OS),darwin)
+    LDFLAGS += -rpath $(CUDA_PATH)/lib
+    CCFLAGS += -arch $(HOST_ARCH)
+else ifeq ($(HOST_ARCH)-$(TARGET_ARCH)-$(TARGET_OS),x86_64-armv7l-linux)
+    LDFLAGS += --dynamic-linker=/lib/ld-linux-armhf.so.3
+    CCFLAGS += -mfloat-abi=hard
+else ifeq ($(TARGET_OS),android)
+    LDFLAGS += -pie
+    CCFLAGS += -fpie -fpic -fexceptions
+endif
+
+ifneq ($(TARGET_ARCH),$(HOST_ARCH))
+    ifeq ($(TARGET_ARCH)-$(TARGET_OS),armv7l-linux)
+        ifneq ($(TARGET_FS),)
+            GCCVERSIONLTEQ46 := $(shell expr `$(HOST_COMPILER) -dumpversion` \<= 4.6)
+            ifeq ($(GCCVERSIONLTEQ46),1)
+                CCFLAGS += --sysroot=$(TARGET_FS)
+            endif
+            LDFLAGS += --sysroot=$(TARGET_FS)
+            LDFLAGS += -rpath-link=$(TARGET_FS)/lib
+            LDFLAGS += -rpath-link=$(TARGET_FS)/usr/lib
+            LDFLAGS += -rpath-link=$(TARGET_FS)/usr/lib/arm-linux-gnueabihf
+        endif
+    endif
+    ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-linux)
+        ifneq ($(TARGET_FS),)
+            GCCVERSIONLTEQ46 := $(shell expr `$(HOST_COMPILER) -dumpversion` \<= 4.6)
+            ifeq ($(GCCVERSIONLTEQ46),1)
+                CCFLAGS += --sysroot=$(TARGET_FS)
+            endif
+            LDFLAGS += --sysroot=$(TARGET_FS)
+            LDFLAGS += -rpath-link=$(TARGET_FS)/lib -L$(TARGET_FS)/lib
+            LDFLAGS += -rpath-link=$(TARGET_FS)/lib/aarch64-linux-gnu -L$(TARGET_FS)/lib/aarch64-linux-gnu
+            LDFLAGS += -rpath-link=$(TARGET_FS)/usr/lib -L$(TARGET_FS)/usr/lib
+            LDFLAGS += -rpath-link=$(TARGET_FS)/usr/lib/aarch64-linux-gnu -L$(TARGET_FS)/usr/lib/aarch64-linux-gnu
+            LDFLAGS += --unresolved-symbols=ignore-in-shared-libs
+            CCFLAGS += -isystem=$(TARGET_FS)/usr/include -I$(TARGET_FS)/usr/include -I$(TARGET_FS)/usr/include/libdrm
+            CCFLAGS += -isystem=$(TARGET_FS)/usr/include/aarch64-linux-gnu -I$(TARGET_FS)/usr/include/aarch64-linux-gnu
+        endif
+    endif
+    ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-qnx)
+        NVCCFLAGS += -D_QNX_SOURCE
+        NVCCFLAGS += --qpp-config 8.3.0,gcc_ntoaarch64le
+        CCFLAGS += -DWIN_INTERFACE_CUSTOM -I/usr/include/aarch64-qnx-gnu
+        LDFLAGS += -lsocket
+        LDFLAGS += -L/usr/lib/aarch64-qnx-gnu
+        CCFLAGS += "-Wl\,-rpath-link\,/usr/lib/aarch64-qnx-gnu"
+        ifdef TARGET_OVERRIDE
+            LDFLAGS += -lslog2
+        endif
+
+        ifneq ($(TARGET_FS),)
+            LDFLAGS += -L$(TARGET_FS)/usr/lib
+            CCFLAGS += "-Wl\,-rpath-link\,$(TARGET_FS)/usr/lib"
+            LDFLAGS += -L$(TARGET_FS)/usr/libnvidia
+            CCFLAGS += "-Wl\,-rpath-link\,$(TARGET_FS)/usr/libnvidia"
+            CCFLAGS += -I$(TARGET_FS)/../include
+        endif
+    endif
+endif
+
+ifdef TARGET_OVERRIDE # cuda toolkit targets override
+    NVCCFLAGS += -target-dir $(TARGET_OVERRIDE)
+endif
+
+# Install directory of different arch
+CUDA_INSTALL_TARGET_DIR :=
+ifeq ($(TARGET_ARCH)-$(TARGET_OS),armv7l-linux)
+    CUDA_INSTALL_TARGET_DIR = targets/armv7-linux-gnueabihf/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-linux)
+    CUDA_INSTALL_TARGET_DIR = targets/aarch64-linux/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),sbsa-linux)
+    CUDA_INSTALL_TARGET_DIR = targets/sbsa-linux/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),armv7l-android)
+    CUDA_INSTALL_TARGET_DIR = targets/armv7-linux-androideabi/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-android)
+    CUDA_INSTALL_TARGET_DIR = targets/aarch64-linux-androideabi/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),armv7l-qnx)
+    CUDA_INSTALL_TARGET_DIR = targets/ARMv7-linux-QNX/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-qnx)
+    CUDA_INSTALL_TARGET_DIR = targets/aarch64-qnx/
+else ifeq ($(TARGET_ARCH),ppc64le)
+    CUDA_INSTALL_TARGET_DIR = targets/ppc64le-linux/
+endif
+
+# Debug build flags
+ifeq ($(dbg),1)
+      NVCCFLAGS += -g -G
+      BUILD_TYPE := debug
+else
+      BUILD_TYPE := release
+endif
+
+ALL_CCFLAGS :=
+ALL_CCFLAGS += $(NVCCFLAGS)
+ALL_CCFLAGS += $(EXTRA_NVCCFLAGS)
+ALL_CCFLAGS += $(addprefix -Xcompiler ,$(CCFLAGS))
+ALL_CCFLAGS += $(addprefix -Xcompiler ,$(EXTRA_CCFLAGS))
+
+SAMPLE_ENABLED := 1
+
+# This sample is not supported on QNX
+ifeq ($(TARGET_OS),qnx)
+  $(info >>> WARNING - UnifiedMemoryPerf is not supported on QNX - waiving sample <<<)
+  SAMPLE_ENABLED := 0
+endif
+
+ALL_LDFLAGS :=
+ALL_LDFLAGS += $(ALL_CCFLAGS)
+ALL_LDFLAGS += $(addprefix -Xlinker ,$(LDFLAGS))
+ALL_LDFLAGS += $(addprefix -Xlinker ,$(EXTRA_LDFLAGS))
+
+# Common includes and paths for CUDA
+INCLUDES  := -I../../../Common
+LIBRARIES :=
+
+################################################################################
+
+# Gencode arguments
+ifeq ($(TARGET_ARCH),$(filter $(TARGET_ARCH),armv7l aarch64 sbsa))
+SMS ?= 53 61 70 72 75 80 86 87
+else
+SMS ?= 35 37 50 52 60 61 70 75 80 86
+endif
+
+ifeq ($(SMS),)
+$(info >>> WARNING - no SM architectures have been specified - waiving sample <<<)
+SAMPLE_ENABLED := 0
+endif
+
+ifeq ($(GENCODE_FLAGS),)
+# Generate SASS code for each SM architecture listed in $(SMS)
+$(foreach sm,$(SMS),$(eval GENCODE_FLAGS += -gencode arch=compute_$(sm),code=sm_$(sm)))
+
+# Generate PTX code from the highest SM architecture in $(SMS) to guarantee forward-compatibility
+HIGHEST_SM := $(lastword $(sort $(SMS)))
+ifneq ($(HIGHEST_SM),)
+GENCODE_FLAGS += -gencode arch=compute_$(HIGHEST_SM),code=compute_$(HIGHEST_SM)
+endif
+endif
+
+ALL_CCFLAGS += --threads 0 --std=c++11
+
+ifeq ($(SAMPLE_ENABLED),0)
+EXEC ?= @echo "[@]"
+endif
+
+################################################################################
+
+# Target rules
+all: build
+
+build: UnifiedMemoryPerf
+
+check.deps:
+ifeq ($(SAMPLE_ENABLED),0)
+	@echo "Sample will be waived due to the above missing dependencies"
+else
+	@echo "Sample is ready - all dependencies have been met"
+endif
+
+commonKernels.o:commonKernels.cu
+	$(EXEC) $(NVCC) $(INCLUDES) $(ALL_CCFLAGS) $(GENCODE_FLAGS) -o $@ -c $<
+
+helperFunctions.o:helperFunctions.cpp
+	$(EXEC) $(NVCC) $(INCLUDES) $(ALL_CCFLAGS) $(GENCODE_FLAGS) -o $@ -c $<
+
+matrixMultiplyPerf.o:matrixMultiplyPerf.cu
+	$(EXEC) $(NVCC) $(INCLUDES) $(ALL_CCFLAGS) $(GENCODE_FLAGS) -o $@ -c $<
+
+UnifiedMemoryPerf: commonKernels.o helperFunctions.o matrixMultiplyPerf.o
+	$(EXEC) $(NVCC) $(ALL_LDFLAGS) $(GENCODE_FLAGS) -o $@ $+ $(LIBRARIES)
+	$(EXEC) mkdir -p ../../../bin/$(TARGET_ARCH)/$(TARGET_OS)/$(BUILD_TYPE)
+	$(EXEC) cp $@ ../../../bin/$(TARGET_ARCH)/$(TARGET_OS)/$(BUILD_TYPE)
+
+run: build
+	$(EXEC) ./UnifiedMemoryPerf
+
+testrun: build
+
+clean:
+	rm -f UnifiedMemoryPerf commonKernels.o helperFunctions.o matrixMultiplyPerf.o
+	rm -rf ../../../bin/$(TARGET_ARCH)/$(TARGET_OS)/$(BUILD_TYPE)/UnifiedMemoryPerf
+
+clobber: clean

Samples/6_Performance/UnifiedMemoryPerf/NsightEclipse.xml

@@ -0,0 +1,100 @@
+<?xml version="1.0" encoding="UTF-8"?> 
+<!DOCTYPE entry SYSTEM "SamplesInfo.dtd">
+<entry>
+  <name>UnifiedMemoryPerf</name>
+  <cuda_api_list>
+    <toolkit>cudaStreamDestroy</toolkit>
+    <toolkit>cudaFree</toolkit>
+    <toolkit>cudaMallocHost</toolkit>
+    <toolkit>cudaMallocManaged</toolkit>
+    <toolkit>cudaMemPrefetchAsync</toolkit>
+    <toolkit>cudaStreamCreate</toolkit>
+    <toolkit>cudaStreamAttachMemAsync</toolkit>
+    <toolkit>cudaFreeHost</toolkit>
+    <toolkit>cudaMalloc</toolkit>
+    <toolkit>cudaMemcpyAsync</toolkit>
+    <toolkit>cudaStreamSynchronize</toolkit>
+    <toolkit>cudaHostGetDevicePointer</toolkit>
+    <toolkit>cudaMemcpy</toolkit>
+    <toolkit>cudaGetDeviceProperties</toolkit>
+  </cuda_api_list>
+  <description><![CDATA[This sample demonstrates the performance comparision using matrix multiplication kernel of Unified Memory with/without hints and other types of memory like zero copy buffers, pageable, pagelocked memory performing synchronous and Asynchronous transfers on a single GPU.]]></description>
+  <devicecompilation>whole</devicecompilation>
+  <includepaths>
+    <path>./</path>
+    <path>../</path>
+    <path>../../../Common</path>
+  </includepaths>
+  <keyconcepts>
+    <concept level="basic">CUDA Systems Integration</concept>
+    <concept level="basic">Unified Memory</concept>
+    <concept level="basic">CUDA Streams and Events</concept>
+    <concept level="basic">Pinned System Paged Memory</concept>
+  </keyconcepts>
+  <keywords>
+    <keyword>CUDA</keyword>
+    <keyword>Unified Memory</keyword>
+    <keyword>Pinned Memory</keyword>
+    <keyword>Zero copy buffer</keyword>
+    <keyword>UVM</keyword>
+    <keyword>Streams</keyword>
+  </keywords>
+  <libraries>
+  </libraries>
+  <librarypaths>
+  </librarypaths>
+  <nsight_eclipse>true</nsight_eclipse>
+  <primary_file>matrixMultiplyPerf.cu</primary_file>
+  <required_dependencies>
+    <dependency>UVM</dependency>
+  </required_dependencies>
+  <scopes>
+    <scope>1:CUDA Basic Topics</scope>
+    <scope>1:CUDA Systems Integration</scope>
+    <scope>1:Unified Memory</scope>
+  </scopes>
+  <sm-arch>sm35</sm-arch>
+  <sm-arch>sm37</sm-arch>
+  <sm-arch>sm50</sm-arch>
+  <sm-arch>sm52</sm-arch>
+  <sm-arch>sm53</sm-arch>
+  <sm-arch>sm60</sm-arch>
+  <sm-arch>sm61</sm-arch>
+  <sm-arch>sm70</sm-arch>
+  <sm-arch>sm72</sm-arch>
+  <sm-arch>sm75</sm-arch>
+  <sm-arch>sm80</sm-arch>
+  <sm-arch>sm86</sm-arch>
+  <sm-arch>sm87</sm-arch>
+  <supported_envs>
+    <env>
+      <arch>x86_64</arch>
+      <platform>linux</platform>
+    </env>
+    <env>
+      <arch>x86_64</arch>
+      <platform>macosx</platform>
+    </env>
+    <env>
+      <platform>windows7</platform>
+    </env>
+    <env>
+      <arch>arm</arch>
+    </env>
+    <env>
+      <arch>sbsa</arch>
+    </env>
+    <env>
+      <arch>aarch64</arch>
+    </env>
+    <env>
+      <arch>ppc64le</arch>
+      <platform>linux</platform>
+    </env>
+  </supported_envs>
+  <supported_sm_architectures>
+    <from>3.5</from>
+  </supported_sm_architectures>
+  <title>Unified and other CUDA Memories Performance</title>
+  <type>exe</type>
+</entry>

Samples/6_Performance/UnifiedMemoryPerf/README.md

@@ -0,0 +1,74 @@
+# UnifiedMemoryPerf - Unified and other CUDA Memories Performance
+
+## Description
+
+This sample demonstrates the performance comparision using matrix multiplication kernel of Unified Memory with/without hints and other types of memory like zero copy buffers, pageable, pagelocked memory performing synchronous and Asynchronous transfers on a single GPU.
+
+## Key Concepts
+
+CUDA Systems Integration, Unified Memory, CUDA Streams and Events, Pinned System Paged Memory
+
+## Supported SM Architectures
+
+[SM 3.5 ](https://developer.nvidia.com/cuda-gpus)  [SM 3.7 ](https://developer.nvidia.com/cuda-gpus)  [SM 5.0 ](https://developer.nvidia.com/cuda-gpus)  [SM 5.2 ](https://developer.nvidia.com/cuda-gpus)  [SM 5.3 ](https://developer.nvidia.com/cuda-gpus)  [SM 6.0 ](https://developer.nvidia.com/cuda-gpus)  [SM 6.1 ](https://developer.nvidia.com/cuda-gpus)  [SM 7.0 ](https://developer.nvidia.com/cuda-gpus)  [SM 7.2 ](https://developer.nvidia.com/cuda-gpus)  [SM 7.5 ](https://developer.nvidia.com/cuda-gpus)  [SM 8.0 ](https://developer.nvidia.com/cuda-gpus)  [SM 8.6 ](https://developer.nvidia.com/cuda-gpus)  [SM 8.7 ](https://developer.nvidia.com/cuda-gpus)
+
+## Supported OSes
+
+Linux, Windows
+
+## Supported CPU Architecture
+
+x86_64, ppc64le, armv7l, aarch64
+
+## CUDA APIs involved
+
+### [CUDA Runtime API](http://docs.nvidia.com/cuda/cuda-runtime-api/index.html)
+cudaStreamDestroy, cudaFree, cudaMallocHost, cudaMallocManaged, cudaMemPrefetchAsync, cudaStreamCreate, cudaStreamAttachMemAsync, cudaFreeHost, cudaMalloc, cudaMemcpyAsync, cudaStreamSynchronize, cudaHostGetDevicePointer, cudaMemcpy, cudaGetDeviceProperties
+
+## Dependencies needed to build/run
+[UVM](../../README.md#uvm)
+
+## Prerequisites
+
+Download and install the [CUDA Toolkit 11.6](https://developer.nvidia.com/cuda-downloads) for your corresponding platform.
+Make sure the dependencies mentioned in [Dependencies]() section above are installed.
+
+## Build and Run
+
+### Windows
+The Windows samples are built using the Visual Studio IDE. Solution files (.sln) are provided for each supported version of Visual Studio, using the format:
+```
+*_vs<version>.sln - for Visual Studio <version>
+```
+Each individual sample has its own set of solution files in its directory:
+
+To build/examine all the samples at once, the complete solution files should be used. To build/examine a single sample, the individual sample solution files should be used.
+> **Note:** Some samples require that the Microsoft DirectX SDK (June 2010 or newer) be installed and that the VC++ directory paths are properly set up (**Tools > Options...**). Check DirectX Dependencies section for details."
+
+### Linux
+The Linux samples are built using makefiles. To use the makefiles, change the current directory to the sample directory you wish to build, and run make:
+```
+$ cd <sample_dir>
+$ make
+```
+The samples makefiles can take advantage of certain options:
+*  **TARGET_ARCH=<arch>** - cross-compile targeting a specific architecture. Allowed architectures are x86_64, ppc64le, armv7l, aarch64.
+    By default, TARGET_ARCH is set to HOST_ARCH. On a x86_64 machine, not setting TARGET_ARCH is the equivalent of setting TARGET_ARCH=x86_64.<br/>
+`$ make TARGET_ARCH=x86_64` <br/> `$ make TARGET_ARCH=ppc64le` <br/> `$ make TARGET_ARCH=armv7l` <br/> `$ make TARGET_ARCH=aarch64` <br/>
+    See [here](http://docs.nvidia.com/cuda/cuda-samples/index.html#cross-samples) for more details.
+*   **dbg=1** - build with debug symbols
+    ```
+    $ make dbg=1
+    ```
+*   **SMS="A B ..."** - override the SM architectures for which the sample will be built, where `"A B ..."` is a space-delimited list of SM architectures. For example, to generate SASS for SM 50 and SM 60, use `SMS="50 60"`.
+    ```
+    $ make SMS="50 60"
+    ```
+
+*  **HOST_COMPILER=<host_compiler>** - override the default g++ host compiler. See the [Linux Installation Guide](http://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html#system-requirements) for a list of supported host compilers.
+```
+    $ make HOST_COMPILER=g++
+```
+
+## References (for more details)
+

Samples/6_Performance/UnifiedMemoryPerf/UnifiedMemoryPerf_vs2017.sln

@@ -0,0 +1,20 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio 2017
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "UnifiedMemoryPerf", "UnifiedMemoryPerf_vs2017.vcxproj", "{997E0757-EA74-4A4E-A0FC-47D8C8831A15}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|x64 = Debug|x64
+		Release|x64 = Release|x64
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Debug|x64.ActiveCfg = Debug|x64
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Debug|x64.Build.0 = Debug|x64
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Release|x64.ActiveCfg = Release|x64
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Release|x64.Build.0 = Release|x64
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+EndGlobal

Samples/6_Performance/UnifiedMemoryPerf/UnifiedMemoryPerf_vs2017.vcxproj

@@ -0,0 +1,115 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <CUDAPropsPath Condition="'$(CUDAPropsPath)'==''">$(VCTargetsPath)\BuildCustomizations</CUDAPropsPath>
+  </PropertyGroup>
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Debug|x64">
+      <Configuration>Debug</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|x64">
+      <Configuration>Release</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>{997E0757-EA74-4A4E-A0FC-47D8C8831A15}</ProjectGuid>
+    <RootNamespace>UnifiedMemoryPerf_vs2017</RootNamespace>
+    <ProjectName>UnifiedMemoryPerf</ProjectName>
+    <CudaToolkitCustomDir />
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(WindowsTargetPlatformVersion)'==''">
+    <LatestTargetPlatformVersion>$([Microsoft.Build.Utilities.ToolLocationHelper]::GetLatestSDKTargetPlatformVersion('Windows', '10.0'))</LatestTargetPlatformVersion>
+    <WindowsTargetPlatformVersion Condition="'$(WindowsTargetPlatformVersion)' == ''">$(LatestTargetPlatformVersion)</WindowsTargetPlatformVersion>
+    <TargetPlatformVersion>$(WindowsTargetPlatformVersion)</TargetPlatformVersion>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <PropertyGroup>
+    <ConfigurationType>Application</ConfigurationType>
+    <CharacterSet>MultiByte</CharacterSet>
+    <PlatformToolset>v141</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)'=='Debug'">
+    <UseDebugLibraries>true</UseDebugLibraries>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)'=='Release'">
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <ImportGroup Label="ExtensionSettings">
+    <Import Project="$(CUDAPropsPath)\CUDA 11.6.props" />
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets">
+    <Import Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" />
+  </ImportGroup>
+  <PropertyGroup Label="UserMacros" />
+  <PropertyGroup>
+    <IntDir>$(Platform)/$(Configuration)/</IntDir>
+    <IncludePath>$(IncludePath)</IncludePath>
+    <CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRules />
+    <CodeAnalysisRuleAssemblies />
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Platform)'=='x64'">
+    <OutDir>../../../bin/win64/$(Configuration)/</OutDir>
+  </PropertyGroup>
+  <ItemDefinitionGroup>
+    <ClCompile>
+      <WarningLevel>Level3</WarningLevel>
+      <PreprocessorDefinitions>WIN32;_MBCS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <AdditionalIncludeDirectories>./;$(CudaToolkitDir)/include;../../../Common;</AdditionalIncludeDirectories>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <AdditionalDependencies>cudart_static.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalLibraryDirectories>$(CudaToolkitLibDir);</AdditionalLibraryDirectories>
+      <OutputFile>$(OutDir)/UnifiedMemoryPerf.exe</OutputFile>
+    </Link>
+    <CudaCompile>
+      <CodeGeneration>compute_35,sm_35;compute_37,sm_37;compute_50,sm_50;compute_52,sm_52;compute_60,sm_60;compute_61,sm_61;compute_70,sm_70;compute_75,sm_75;compute_80,sm_80;compute_86,sm_86;</CodeGeneration>
+      <AdditionalOptions>-Xcompiler "/wd 4819"  --threads 0 </AdditionalOptions>
+      <Include>./;../../../Common</Include>
+      <Defines>WIN32</Defines>
+    </CudaCompile>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)'=='Debug'">
+    <ClCompile>
+      <Optimization>Disabled</Optimization>
+      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <LinkTimeCodeGeneration>Default</LinkTimeCodeGeneration>
+    </Link>
+    <CudaCompile>
+      <Runtime>MTd</Runtime>
+      <TargetMachinePlatform>64</TargetMachinePlatform>
+    </CudaCompile>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)'=='Release'">
+    <ClCompile>
+      <Optimization>MaxSpeed</Optimization>
+      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <GenerateDebugInformation>false</GenerateDebugInformation>
+      <LinkTimeCodeGeneration>UseLinkTimeCodeGeneration</LinkTimeCodeGeneration>
+    </Link>
+    <CudaCompile>
+      <Runtime>MT</Runtime>
+      <TargetMachinePlatform>64</TargetMachinePlatform>
+    </CudaCompile>
+  </ItemDefinitionGroup>
+  <ItemGroup>
+    <CudaCompile Include="commonKernels.cu" />
+    <ClCompile Include="helperFunctions.cpp" />
+    <CudaCompile Include="matrixMultiplyPerf.cu" />
+    <ClInclude Include="commonDefs.hpp" />
+    <ClInclude Include="commonKernels.hpp" />
+  </ItemGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <ImportGroup Label="ExtensionTargets">
+    <Import Project="$(CUDAPropsPath)\CUDA 11.6.targets" />
+  </ImportGroup>
+</Project>

Samples/6_Performance/UnifiedMemoryPerf/UnifiedMemoryPerf_vs2019.sln

@@ -0,0 +1,20 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio 2019
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "UnifiedMemoryPerf", "UnifiedMemoryPerf_vs2019.vcxproj", "{997E0757-EA74-4A4E-A0FC-47D8C8831A15}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|x64 = Debug|x64
+		Release|x64 = Release|x64
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Debug|x64.ActiveCfg = Debug|x64
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Debug|x64.Build.0 = Debug|x64
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Release|x64.ActiveCfg = Release|x64
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Release|x64.Build.0 = Release|x64
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+EndGlobal

Samples/6_Performance/UnifiedMemoryPerf/UnifiedMemoryPerf_vs2019.vcxproj

@@ -0,0 +1,111 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <CUDAPropsPath Condition="'$(CUDAPropsPath)'==''">$(VCTargetsPath)\BuildCustomizations</CUDAPropsPath>
+  </PropertyGroup>
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Debug|x64">
+      <Configuration>Debug</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|x64">
+      <Configuration>Release</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>{997E0757-EA74-4A4E-A0FC-47D8C8831A15}</ProjectGuid>
+    <RootNamespace>UnifiedMemoryPerf_vs2019</RootNamespace>
+    <ProjectName>UnifiedMemoryPerf</ProjectName>
+    <CudaToolkitCustomDir />
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <PropertyGroup>
+    <ConfigurationType>Application</ConfigurationType>
+    <CharacterSet>MultiByte</CharacterSet>
+    <PlatformToolset>v142</PlatformToolset>
+    <WindowsTargetPlatformVersion>10.0</WindowsTargetPlatformVersion>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)'=='Debug'">
+    <UseDebugLibraries>true</UseDebugLibraries>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)'=='Release'">
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <ImportGroup Label="ExtensionSettings">
+    <Import Project="$(CUDAPropsPath)\CUDA 11.6.props" />
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets">
+    <Import Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" />
+  </ImportGroup>
+  <PropertyGroup Label="UserMacros" />
+  <PropertyGroup>
+    <IntDir>$(Platform)/$(Configuration)/</IntDir>
+    <IncludePath>$(IncludePath)</IncludePath>
+    <CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRules />
+    <CodeAnalysisRuleAssemblies />
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Platform)'=='x64'">
+    <OutDir>../../../bin/win64/$(Configuration)/</OutDir>
+  </PropertyGroup>
+  <ItemDefinitionGroup>
+    <ClCompile>
+      <WarningLevel>Level3</WarningLevel>
+      <PreprocessorDefinitions>WIN32;_MBCS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <AdditionalIncludeDirectories>./;$(CudaToolkitDir)/include;../../../Common;</AdditionalIncludeDirectories>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <AdditionalDependencies>cudart_static.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalLibraryDirectories>$(CudaToolkitLibDir);</AdditionalLibraryDirectories>
+      <OutputFile>$(OutDir)/UnifiedMemoryPerf.exe</OutputFile>
+    </Link>
+    <CudaCompile>
+      <CodeGeneration>compute_35,sm_35;compute_37,sm_37;compute_50,sm_50;compute_52,sm_52;compute_60,sm_60;compute_61,sm_61;compute_70,sm_70;compute_75,sm_75;compute_80,sm_80;compute_86,sm_86;</CodeGeneration>
+      <AdditionalOptions>-Xcompiler "/wd 4819"  --threads 0 </AdditionalOptions>
+      <Include>./;../../../Common</Include>
+      <Defines>WIN32</Defines>
+    </CudaCompile>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)'=='Debug'">
+    <ClCompile>
+      <Optimization>Disabled</Optimization>
+      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <LinkTimeCodeGeneration>Default</LinkTimeCodeGeneration>
+    </Link>
+    <CudaCompile>
+      <Runtime>MTd</Runtime>
+      <TargetMachinePlatform>64</TargetMachinePlatform>
+    </CudaCompile>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)'=='Release'">
+    <ClCompile>
+      <Optimization>MaxSpeed</Optimization>
+      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <GenerateDebugInformation>false</GenerateDebugInformation>
+      <LinkTimeCodeGeneration>UseLinkTimeCodeGeneration</LinkTimeCodeGeneration>
+    </Link>
+    <CudaCompile>
+      <Runtime>MT</Runtime>
+      <TargetMachinePlatform>64</TargetMachinePlatform>
+    </CudaCompile>
+  </ItemDefinitionGroup>
+  <ItemGroup>
+    <CudaCompile Include="commonKernels.cu" />
+    <ClCompile Include="helperFunctions.cpp" />
+    <CudaCompile Include="matrixMultiplyPerf.cu" />
+    <ClInclude Include="commonDefs.hpp" />
+    <ClInclude Include="commonKernels.hpp" />
+  </ItemGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <ImportGroup Label="ExtensionTargets">
+    <Import Project="$(CUDAPropsPath)\CUDA 11.6.targets" />
+  </ImportGroup>
+</Project>

Samples/6_Performance/UnifiedMemoryPerf/UnifiedMemoryPerf_vs2022.sln

@@ -0,0 +1,20 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio 2022
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "UnifiedMemoryPerf", "UnifiedMemoryPerf_vs2022.vcxproj", "{997E0757-EA74-4A4E-A0FC-47D8C8831A15}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|x64 = Debug|x64
+		Release|x64 = Release|x64
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Debug|x64.ActiveCfg = Debug|x64
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Debug|x64.Build.0 = Debug|x64
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Release|x64.ActiveCfg = Release|x64
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Release|x64.Build.0 = Release|x64
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+EndGlobal

Samples/6_Performance/UnifiedMemoryPerf/UnifiedMemoryPerf_vs2022.vcxproj

@@ -0,0 +1,111 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <CUDAPropsPath Condition="'$(CUDAPropsPath)'==''">$(VCTargetsPath)\BuildCustomizations</CUDAPropsPath>
+  </PropertyGroup>
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Debug|x64">
+      <Configuration>Debug</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|x64">
+      <Configuration>Release</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>{997E0757-EA74-4A4E-A0FC-47D8C8831A15}</ProjectGuid>
+    <RootNamespace>UnifiedMemoryPerf_vs2022</RootNamespace>
+    <ProjectName>UnifiedMemoryPerf</ProjectName>
+    <CudaToolkitCustomDir />
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <PropertyGroup>
+    <ConfigurationType>Application</ConfigurationType>
+    <CharacterSet>MultiByte</CharacterSet>
+    <PlatformToolset>v143</PlatformToolset>
+    <WindowsTargetPlatformVersion>10.0</WindowsTargetPlatformVersion>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)'=='Debug'">
+    <UseDebugLibraries>true</UseDebugLibraries>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)'=='Release'">
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <ImportGroup Label="ExtensionSettings">
+    <Import Project="$(CUDAPropsPath)\CUDA 11.6.props" />
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets">
+    <Import Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" />
+  </ImportGroup>
+  <PropertyGroup Label="UserMacros" />
+  <PropertyGroup>
+    <IntDir>$(Platform)/$(Configuration)/</IntDir>
+    <IncludePath>$(IncludePath)</IncludePath>
+    <CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRules />
+    <CodeAnalysisRuleAssemblies />
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Platform)'=='x64'">
+    <OutDir>../../../bin/win64/$(Configuration)/</OutDir>
+  </PropertyGroup>
+  <ItemDefinitionGroup>
+    <ClCompile>
+      <WarningLevel>Level3</WarningLevel>
+      <PreprocessorDefinitions>WIN32;_MBCS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <AdditionalIncludeDirectories>./;$(CudaToolkitDir)/include;../../../Common;</AdditionalIncludeDirectories>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <AdditionalDependencies>cudart_static.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalLibraryDirectories>$(CudaToolkitLibDir);</AdditionalLibraryDirectories>
+      <OutputFile>$(OutDir)/UnifiedMemoryPerf.exe</OutputFile>
+    </Link>
+    <CudaCompile>
+      <CodeGeneration>compute_35,sm_35;compute_37,sm_37;compute_50,sm_50;compute_52,sm_52;compute_60,sm_60;compute_61,sm_61;compute_70,sm_70;compute_75,sm_75;compute_80,sm_80;compute_86,sm_86;</CodeGeneration>
+      <AdditionalOptions>-Xcompiler "/wd 4819"  --threads 0 </AdditionalOptions>
+      <Include>./;../../../Common</Include>
+      <Defines>WIN32</Defines>
+    </CudaCompile>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)'=='Debug'">
+    <ClCompile>
+      <Optimization>Disabled</Optimization>
+      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <LinkTimeCodeGeneration>Default</LinkTimeCodeGeneration>
+    </Link>
+    <CudaCompile>
+      <Runtime>MTd</Runtime>
+      <TargetMachinePlatform>64</TargetMachinePlatform>
+    </CudaCompile>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)'=='Release'">
+    <ClCompile>
+      <Optimization>MaxSpeed</Optimization>
+      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <GenerateDebugInformation>false</GenerateDebugInformation>
+      <LinkTimeCodeGeneration>UseLinkTimeCodeGeneration</LinkTimeCodeGeneration>
+    </Link>
+    <CudaCompile>
+      <Runtime>MT</Runtime>
+      <TargetMachinePlatform>64</TargetMachinePlatform>
+    </CudaCompile>
+  </ItemDefinitionGroup>
+  <ItemGroup>
+    <CudaCompile Include="commonKernels.cu" />
+    <ClCompile Include="helperFunctions.cpp" />
+    <CudaCompile Include="matrixMultiplyPerf.cu" />
+    <ClInclude Include="commonDefs.hpp" />
+    <ClInclude Include="commonKernels.hpp" />
+  </ItemGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <ImportGroup Label="ExtensionTargets">
+    <Import Project="$(CUDAPropsPath)\CUDA 11.6.targets" />
+  </ImportGroup>
+</Project>

Samples/6_Performance/UnifiedMemoryPerf/commonDefs.hpp

@@ -0,0 +1,88 @@
+/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _COMMON_DEFS_
+#define _COMMON_DEFS_
+#include <cuda.h>
+
+#define ONE_KB 1024
+#define ONE_MB (ONE_KB * ONE_KB)
+
+extern size_t maxSampleSizeInMb;
+extern int numKernelRuns;
+extern int verboseResults;
+
+extern unsigned int findNumSizesToTest(unsigned int minSize,
+                                       unsigned int maxSize,
+                                       unsigned int multiplier);
+
+// For Tracking the different memory allocation types
+typedef enum memAllocType_enum {
+  MEMALLOC_TYPE_START,
+  USE_MANAGED_MEMORY_WITH_HINTS = MEMALLOC_TYPE_START,
+  USE_MANAGED_MEMORY_WITH_HINTS_ASYNC,
+  USE_MANAGED_MEMORY,
+  USE_ZERO_COPY,
+  USE_HOST_PAGEABLE_AND_DEVICE_MEMORY,
+  USE_HOST_PAGEABLE_AND_DEVICE_MEMORY_ASYNC,
+  USE_HOST_PAGELOCKED_AND_DEVICE_MEMORY,
+  USE_HOST_PAGELOCKED_AND_DEVICE_MEMORY_ASYNC,
+  MEMALLOC_TYPE_END = USE_HOST_PAGELOCKED_AND_DEVICE_MEMORY_ASYNC,
+  MEMALLOC_TYPE_INVALID,
+  MEMALLOC_TYPE_COUNT = MEMALLOC_TYPE_INVALID
+} MemAllocType;
+
+typedef enum bandwidthType_enum {
+  READ_BANDWIDTH,
+  WRITE_BANDWIDTH
+} BandwidthType;
+
+extern const char *memAllocTypeStr[];
+extern const char *memAllocTypeShortStr[];
+
+struct resultsData;
+struct testResults;
+
+void createAndInitTestResults(struct testResults **results,
+                              const char *testName,
+                              unsigned int numMeasurements,
+                              unsigned int numSizesToTest);
+unsigned long *getPtrSizesToTest(struct testResults *results);
+
+void freeTestResultsAndAllResultsData(struct testResults *results);
+
+void createResultDataAndAddToTestResults(struct resultsData **ptrData,
+                                         struct testResults *results,
+                                         const char *resultsName,
+                                         bool printOnlyInVerbose,
+                                         bool reportAsBandwidth);
+double *getPtrRunTimesInMs(struct resultsData *data, int allocType,
+                           int sizeIndex);
+
+void printResults(struct testResults *results,
+                  bool print_launch_transfer_results, bool print_std_deviation);
+#endif

Samples/6_Performance/UnifiedMemoryPerf/commonKernels.cu

@@ -0,0 +1,33 @@
+/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "commonKernels.hpp"
+
+__global__ void spinWhileLessThanOne(volatile unsigned int *latch) {
+  while (latch[0] < 1)
+    ;
+}

Samples/6_Performance/UnifiedMemoryPerf/commonKernels.hpp

@@ -0,0 +1,28 @@
+/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+__global__ void spinWhileLessThanOne(volatile unsigned int *latch);

Samples/6_Performance/UnifiedMemoryPerf/helperFunctions.cpp

@@ -0,0 +1,303 @@
+/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <stdio.h>
+#include <string.h>
+#include "commonDefs.hpp"
+#define CU_INIT_UUID
+#include <cmath>
+
+#define UNITS_Time "ms"
+#define UNITS_BW "MB/s"
+#define KB_str "KB"
+#define MB_str "MB"
+
+struct resultsData {
+  char resultsName[64];
+  struct testResults *results;
+  // this has MEMALLOC_TYPE_COUNT * results->numSizesToTest *
+  // results->numMeasurements elements
+  double **runTimesInMs[MEMALLOC_TYPE_COUNT];
+  double *averageRunTimesInMs[MEMALLOC_TYPE_COUNT];
+  double *stdDevRunTimesInMs[MEMALLOC_TYPE_COUNT];
+  double *stdDevBandwidthInMBps[MEMALLOC_TYPE_COUNT];
+  bool printOnlyInVerbose;
+  bool reportAsBandwidth;
+  struct resultsData *next;
+};
+
+struct testResults {
+  char testName[64];
+  unsigned int numMeasurements;
+  unsigned long *sizesToTest;
+  unsigned int numSizesToTest;
+  struct resultsData *resultsDataHead;
+  struct resultsData *resultsDataTail;
+};
+
+unsigned int findNumSizesToTest(unsigned int minSize, unsigned int maxSize,
+                                unsigned int multiplier) {
+  unsigned int numSizesToTest = 0;
+  while (minSize <= maxSize) {
+    numSizesToTest++;
+    minSize *= multiplier;
+  }
+  return numSizesToTest;
+}
+
+int compareDoubles(const void *ptr1, const void *ptr2) {
+  return (*(double *)ptr1 > *(double *)ptr2) ? 1 : -1;
+}
+
+static inline double getTimeOrBandwidth(double runTimeInMs, unsigned long size,
+                                        bool getBandwidth) {
+  return (getBandwidth) ? (1000 * (size / runTimeInMs)) / ONE_MB : runTimeInMs;
+}
+
+void createAndInitTestResults(struct testResults **ptrResults,
+                              const char *testName,
+                              unsigned int numMeasurements,
+                              unsigned int numSizesToTest) {
+  unsigned int i;
+  struct testResults *results;
+  results = (struct testResults *)malloc(sizeof(struct testResults));
+  memset(results, 0, sizeof(struct testResults));
+  strcpy(results->testName, testName);
+  results->numMeasurements = numMeasurements;
+  results->numSizesToTest = numSizesToTest;
+  results->sizesToTest =
+      (unsigned long *)malloc(numSizesToTest * sizeof(unsigned long));
+  results->resultsDataHead = NULL;
+  results->resultsDataTail = NULL;
+
+  *ptrResults = results;
+}
+
+unsigned long *getPtrSizesToTest(struct testResults *results) {
+  return results->sizesToTest;
+}
+
+void createResultDataAndAddToTestResults(struct resultsData **ptrData,
+                                         struct testResults *results,
+                                         const char *resultsName,
+                                         bool printOnlyInVerbose,
+                                         bool reportAsBandwidth) {
+  unsigned int i, j;
+  struct resultsData *data;
+  data = (struct resultsData *)malloc(sizeof(struct resultsData));
+  memset(data, 0, sizeof(struct resultsData));
+  strcpy(data->resultsName, resultsName);
+  data->results = results;
+  for (i = 0; i < MEMALLOC_TYPE_COUNT; i++) {
+    data->runTimesInMs[i] =
+        (double **)malloc(results->numSizesToTest * sizeof(double *));
+    for (j = 0; j < results->numSizesToTest; j++) {
+      data->runTimesInMs[i][j] =
+          (double *)malloc(results->numMeasurements * sizeof(double));
+    }
+    data->averageRunTimesInMs[i] =
+        (double *)malloc(results->numSizesToTest * sizeof(double));
+    data->stdDevRunTimesInMs[i] =
+        (double *)malloc(results->numSizesToTest * sizeof(double));
+    data->stdDevBandwidthInMBps[i] =
+        (double *)malloc(results->numSizesToTest * sizeof(double));
+  }
+  data->printOnlyInVerbose = printOnlyInVerbose;
+  data->reportAsBandwidth = reportAsBandwidth;
+  data->next = NULL;
+  *ptrData = data;
+  if (results->resultsDataHead == NULL) {
+    results->resultsDataHead = data;
+    results->resultsDataTail = data;
+  } else {
+    results->resultsDataTail->next = data;
+    results->resultsDataTail = data;
+  }
+}
+
+double *getPtrRunTimesInMs(struct resultsData *data, int allocType,
+                           int sizeIndex) {
+  return data->runTimesInMs[allocType][sizeIndex];
+}
+
+void freeTestResultsAndAllResultsData(struct testResults *results) {
+  struct resultsData *data, *dataToFree;
+  unsigned int i, j;
+  for (data = results->resultsDataHead; data != NULL;) {
+    for (i = 0; i < MEMALLOC_TYPE_COUNT; i++) {
+      for (j = 0; j < results->numSizesToTest; j++) {
+        free(data->runTimesInMs[i][j]);
+      }
+      free(data->runTimesInMs[i]);
+      free(data->averageRunTimesInMs[i]);
+      free(data->stdDevRunTimesInMs[i]);
+      free(data->stdDevBandwidthInMBps[i]);
+    }
+    dataToFree = data;
+    data = data->next;
+    free(dataToFree);
+  }
+  free(results->sizesToTest);
+  free(results);
+}
+
+void calculateAverageAndStdDev(double *pAverage, double *pStdDev,
+                               double *allResults, unsigned int count) {
+  unsigned int i;
+  double average = 0.0;
+  double stdDev = 0.0;
+  for (i = 0; i < count; i++) {
+    average += allResults[i];
+  }
+  average /= count;
+  for (i = 0; i < count; i++) {
+    stdDev += (allResults[i] - average) * (allResults[i] - average);
+  }
+  stdDev /= count;
+  stdDev = sqrt(stdDev);
+  *pAverage = average;
+  *pStdDev = (average == 0.0) ? 0.0 : ((100.0 * stdDev) / average);
+}
+
+void calculateStdDevBandwidth(double *pStdDev, double *allResults,
+                              unsigned int count, unsigned long size) {
+  unsigned int i;
+  double bandwidth;
+  double average = 0.0;
+  double stdDev = 0.0;
+  for (i = 0; i < count; i++) {
+    bandwidth = (1000 * (size / allResults[i])) / ONE_MB;
+    average += bandwidth;
+  }
+  average /= count;
+  for (i = 0; i < count; i++) {
+    bandwidth = (1000 * (size / allResults[i])) / ONE_MB;
+    stdDev += (bandwidth - average) * (bandwidth - average);
+  }
+  stdDev /= count;
+  stdDev = sqrt(stdDev);
+  *pStdDev = (average == 0.0) ? 0.0 : ((100.0 * stdDev) / average);
+}
+
+void printTimesInTableFormat(struct testResults *results,
+                             struct resultsData *data, bool printAverage,
+                             bool printStdDev) {
+  unsigned int i, j;
+  bool printStdDevBandwidth = printStdDev && data->reportAsBandwidth;
+  printf("Size_KB");
+  for (i = 0; i < MEMALLOC_TYPE_COUNT; i++) {
+    printf("\t%7s", memAllocTypeShortStr[i]);
+  }
+  printf("\n");
+  for (j = 0; j < results->numSizesToTest; j++) {
+    printf("%lu", results->sizesToTest[j] / ONE_KB);
+    for (i = 0; i < MEMALLOC_TYPE_COUNT; i++) {
+      printf(data->reportAsBandwidth ? "\t%7.2lf" : "\t%7.3lf",
+             printStdDevBandwidth
+                 ? data->stdDevBandwidthInMBps[i][j]
+                 : getTimeOrBandwidth(
+                       printAverage ? data->averageRunTimesInMs[i][j]
+                                    : data->stdDevRunTimesInMs[i][j],
+                       results->sizesToTest[j], data->reportAsBandwidth));
+    }
+    printf("\n");
+  }
+}
+
+void printAllResultsInVerboseMode(struct testResults *results,
+                                  struct resultsData *data) {
+  unsigned int i, j, k;
+  for (i = 0; i < MEMALLOC_TYPE_COUNT; i++) {
+    printf("Verbose mode, printing all results for %s\n", memAllocTypeStr[i]);
+    printf("Instance");
+    for (j = 0; j < results->numSizesToTest; j++) {
+      printf("\t%lu", results->sizesToTest[j] / ONE_KB);
+    }
+    printf("\n");
+    for (k = 0; k < results->numMeasurements; k++) {
+      printf("%u", k);
+      for (j = 0; j < results->numSizesToTest; j++) {
+        printf(data->reportAsBandwidth ? "\t%7.2lf" : "\t%7.3lf",
+               getTimeOrBandwidth(data->runTimesInMs[i][j][k],
+                                  results->sizesToTest[j],
+                                  data->reportAsBandwidth));
+      }
+      printf("\n");
+    }
+  }
+}
+
+void printResults(struct testResults *results,
+                  bool print_launch_transfer_results,
+                  bool print_std_deviation) {
+  char vulcanPrint[256];
+  char resultNameNoSpaces[64];
+  unsigned int i, j, k;
+  struct resultsData *resultsIter;
+  bool sizeGreaterThan1MB;
+  for (resultsIter = results->resultsDataHead; resultsIter != NULL;
+       resultsIter = resultsIter->next) {
+    if (!verboseResults && resultsIter->printOnlyInVerbose) {
+      continue;
+    }
+    if (!print_launch_transfer_results) {
+      if (!(strcmp(resultsIter->resultsName, "Overall Time") == 0)) {
+        continue;
+      }
+    }
+    // regular print
+    printf("\n%s For %s ", resultsIter->resultsName, results->testName);
+    printf("\n");
+    for (j = 0; j < results->numSizesToTest; j++) {
+      for (i = 0; i < MEMALLOC_TYPE_COUNT; i++) {
+        calculateAverageAndStdDev(&resultsIter->averageRunTimesInMs[i][j],
+                                  &resultsIter->stdDevRunTimesInMs[i][j],
+                                  resultsIter->runTimesInMs[i][j],
+                                  results->numMeasurements);
+        if (resultsIter->reportAsBandwidth) {
+          calculateStdDevBandwidth(&resultsIter->stdDevBandwidthInMBps[i][j],
+                                   resultsIter->runTimesInMs[i][j],
+                                   results->numMeasurements,
+                                   results->sizesToTest[j]);
+        }
+      }
+    }
+    printf("\nPrinting Average of %u measurements in (%s)\n",
+           results->numMeasurements,
+           resultsIter->reportAsBandwidth ? UNITS_BW : UNITS_Time);
+    printTimesInTableFormat(results, resultsIter, true, false);
+    if (print_std_deviation) {
+      printf(
+          "\nPrinting Standard Deviation as %% of average of %u measurements\n",
+          results->numMeasurements);
+      printTimesInTableFormat(results, resultsIter, false, true);
+    }
+    if (verboseResults) {
+      printAllResultsInVerboseMode(results, resultsIter);
+    }
+  }
+}

Samples/6_Performance/UnifiedMemoryPerf/matrixMultiplyPerf.cu

@@ -0,0 +1,697 @@
+/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <helper_cuda.h>
+#include <helper_timer.h>
+#include "commonDefs.hpp"
+#include "commonKernels.hpp"
+
+#define VERIFY_GPU_CORRECTNESS 0
+
+size_t maxSampleSizeInMb = 64;
+int numKernelRuns = 20;
+int verboseResults = 0;
+
+const char *memAllocTypeStr[MEMALLOC_TYPE_COUNT] = {
+    "Managed_Memory_With_Hints",
+    "Managed_Memory_With_Hints_FullyAsync",
+    "Managed_Memory_NoHints",
+    "Zero_Copy",
+    "Memcpy_HostMalloc_DeviceCudaMalloc",
+    "MemcpyAsync_HostMalloc_DeviceCudaMalloc",
+    "Memcpy_HostCudaHostAlloc_DeviceCudaMalloc",
+    "MemcpyAsync_HostCudaHostAlloc_DeviceCudaMalloc"};
+
+const char *memAllocTypeShortStr[MEMALLOC_TYPE_COUNT] = {
+    "UMhint",   // Managed Memory With Hints
+    "UMhntAs",  // Managed Memory With_Hints Async
+    "UMeasy",   // Managed_Memory with No Hints
+    "0Copy",    // Zero Copy
+    "MemCopy",  // USE HOST PAGEABLE AND DEVICE_MEMORY
+    "CpAsync",  // USE HOST PAGEABLE AND DEVICE_MEMORY ASYNC
+    "CpHpglk",  // USE HOST PAGELOCKED AND DEVICE MEMORY
+    "CpPglAs"   // USE HOST PAGELOCKED AND DEVICE MEMORY ASYNC
+};
+
+static float RandFloat(float low, float high) {
+  float t = (float)rand() / (float)RAND_MAX;
+  return (1.0f - t) * low + t * high;
+}
+
+void fillMatrixWithRandomValues(float *matrix, unsigned int matrixDim) {
+  unsigned int i, j;
+  for (i = 0; i < matrixDim; ++i) {
+    for (j = 0; j < matrixDim; ++j) {
+      matrix[j + i * matrixDim] = RandFloat(0.0f, 10.0f);
+    }
+  }
+}
+
+#if VERIFY_GPU_CORRECTNESS
+void verifyMatrixMultiplyCorrectness(float *C, float *A, float *B,
+                                     unsigned int matrixDim) {
+  unsigned int i, j, k, numErrors = 0;
+  for (i = 0; i < matrixDim; ++i) {
+    for (j = 0; j < matrixDim; ++j) {
+      float result = 0.0f;
+      for (k = 0; k < matrixDim; ++k) {
+        result += A[k + i * matrixDim] * B[j + k * matrixDim];
+      }
+      if (fabs(C[j + i * matrixDim] - result) > 0.001 * matrixDim) {
+        printf("At [%u, %u]: Expected %f, Found %f\n", i, j, result,
+               C[j + i * matrixDim]);
+        ++numErrors;
+      }
+    }
+  }
+  if (numErrors != 0) {
+    printf("%d value mismatches occured\n", numErrors);
+    fflush(stdout);
+    exit(EXIT_FAILURE);  // exit since value mismatches occured
+  }
+}
+#endif
+
+void copyMatrix(float *dstMatrix, float *srcMatrix, unsigned int matrixDim) {
+  size_t size = matrixDim * matrixDim * sizeof(float);
+  memcpy(dstMatrix, srcMatrix, size);
+}
+
+void verifyMatrixData(float *expectedData, float *observedData,
+                      unsigned int matrixDim) {
+  unsigned int i, j, numErrors = 0;
+  for (i = 0; i < matrixDim; ++i) {
+    for (j = 0; j < matrixDim; ++j) {
+      if (expectedData[j + i * matrixDim] != observedData[j + i * matrixDim]) {
+        ++numErrors;
+        if (verboseResults) {
+          printf("At [%u, %u]: Expected %f, Found %f\n", i, j,
+                 expectedData[j + i * matrixDim],
+                 observedData[j + i * matrixDim]);
+        }
+      }
+    }
+  }
+  if (numErrors != 0) {
+    printf("%d value mismatches occured\n", numErrors);
+    fflush(stdout);
+    exit(EXIT_FAILURE);  // exit since value mismatches occured
+  }
+}
+
+#define BLOCK_SIZE 32
+__global__ void matrixMultiplyKernel(float *C, float *A, float *B,
+                                     unsigned int matrixDim) {
+  // Block index
+  int bx = blockIdx.x;
+  int by = blockIdx.y;
+
+  // Thread index
+  int tx = threadIdx.x;
+  int ty = threadIdx.y;
+
+  unsigned int wA = matrixDim;
+  unsigned int wB = matrixDim;
+
+  // Index of the first sub-matrix of A processed by the block
+  int aBegin = matrixDim * BLOCK_SIZE * by;
+
+  // Index of the last sub-matrix of A processed by the block
+  int aEnd = aBegin + wA - 1;
+
+  // Step size used to iterate through the sub-matrices of A
+  int aStep = BLOCK_SIZE;
+
+  // Index of the first sub-matrix of B processed by the block
+  int bBegin = BLOCK_SIZE * bx;
+
+  // Step size used to iterate through the sub-matrices of B
+  int bStep = BLOCK_SIZE * wB;
+
+  // Csub is used to store the element of the block sub-matrix
+  // that is computed by the thread
+  float Csub = 0;
+
+  // Loop over all the sub-matrices of A and B
+  // required to compute the block sub-matrix
+  for (int a = aBegin, b = bBegin; a <= aEnd; a += aStep, b += bStep) {
+    // Declaration of the shared memory array As used to
+    // store the sub-matrix of A
+    __shared__ float As[BLOCK_SIZE][BLOCK_SIZE];
+
+    // Declaration of the shared memory array Bs used to
+    // store the sub-matrix of B
+    __shared__ float Bs[BLOCK_SIZE][BLOCK_SIZE];
+
+    // Load the matrices from device memory
+    // to shared memory; each thread loads
+    // one element of each matrix
+    As[ty][tx] = A[a + wA * ty + tx];
+    Bs[ty][tx] = B[b + wB * ty + tx];
+
+    // Synchronize to make sure the matrices are loaded
+    __syncthreads();
+
+    // Multiply the two matrices together;
+    // each thread computes one element
+    // of the block sub-matrix
+#pragma unroll
+
+    for (int k = 0; k < BLOCK_SIZE; ++k) {
+      Csub += As[ty][k] * Bs[k][tx];
+    }
+
+    // Synchronize to make sure that the preceding
+    // computation is done before loading two new
+    // sub-matrices of A and B in the next iteration
+    __syncthreads();
+  }
+
+  // Write the block sub-matrix to device memory;
+  // each thread writes one element
+  int c = wB * BLOCK_SIZE * by + BLOCK_SIZE * bx;
+  C[c + wB * ty + tx] = Csub;
+}
+
+void runMatrixMultiplyKernel(unsigned int matrixDim, int allocType,
+                             unsigned int numLoops, double *gpuLaunchCallsTimes,
+                             double *gpuTransferToCallsTimes,
+                             double *gpuTransferFromCallsTimes,
+                             double *gpuLaunchAndTransferCallsTimes,
+                             double *gpuLaunchTransferSyncTimes,
+                             double *cpuAccessTimes, double *overallTimes,
+                             int device_id) {
+  float *dptrA = NULL, *hptrA = NULL;
+  float *dptrB = NULL, *hptrB = NULL;
+  float *dptrC = NULL, *hptrC = NULL;
+  float *randValuesX = NULL, *randValuesY = NULL;
+  float *randValuesVerifyXmulY = NULL, *randValuesVerifyYmulX = NULL;
+  bool copyRequired = false, hintsRequired = false;
+  bool someTransferOpRequired;
+  bool isAsync = false;
+  cudaStream_t streamToRunOn;
+  unsigned int *latch;
+  size_t size = matrixDim * matrixDim * sizeof(float);
+  dim3 threads(32, 32);
+  dim3 grid(matrixDim / threads.x, matrixDim / threads.y);
+  StopWatchInterface *gpuLaunchCallsTimer = 0, *gpuTransferCallsTimer = 0;
+  StopWatchInterface *gpuSyncTimer = 0, *cpuAccessTimer = 0;
+  sdkCreateTimer(&gpuLaunchCallsTimer);
+  sdkCreateTimer(&gpuTransferCallsTimer);
+  sdkCreateTimer(&gpuSyncTimer);
+  sdkCreateTimer(&cpuAccessTimer);
+  unsigned int i;
+
+  cudaDeviceProp deviceProp;
+  checkCudaErrors(cudaGetDeviceProperties(&deviceProp, device_id));
+  checkCudaErrors(cudaStreamCreate(&streamToRunOn));
+
+  randValuesX = (float *)malloc(size);
+  if (!randValuesX) {
+    exit(EXIT_FAILURE);  // exit since memory allocation error
+  }
+  randValuesY = (float *)malloc(size);
+  if (!randValuesY) {
+    exit(EXIT_FAILURE);  // exit since memory allocation error
+  }
+  randValuesVerifyXmulY = (float *)malloc(size);
+  if (!randValuesVerifyXmulY) {
+    exit(EXIT_FAILURE);  // exit since memory allocation error
+  }
+  randValuesVerifyYmulX = (float *)malloc(size);
+  if (!randValuesVerifyYmulX) {
+    exit(EXIT_FAILURE);  // exit since memory allocation error
+  }
+  checkCudaErrors(cudaMalloc(&dptrA, size));
+  checkCudaErrors(cudaMalloc(&dptrB, size));
+  checkCudaErrors(cudaMalloc(&dptrC, size));
+
+  fillMatrixWithRandomValues(randValuesX, matrixDim);
+  fillMatrixWithRandomValues(randValuesY, matrixDim);
+
+  checkCudaErrors(
+      cudaMemcpyAsync(dptrA, randValuesX, size, cudaMemcpyHostToDevice));
+  checkCudaErrors(
+      cudaMemcpyAsync(dptrB, randValuesY, size, cudaMemcpyHostToDevice));
+  matrixMultiplyKernel<<<grid, threads>>>(dptrC, dptrA, dptrB, matrixDim);
+  checkCudaErrors(cudaMemcpyAsync(randValuesVerifyXmulY, dptrC, size,
+                                  cudaMemcpyDeviceToHost));
+  checkCudaErrors(cudaStreamSynchronize(NULL));
+  matrixMultiplyKernel<<<grid, threads>>>(dptrC, dptrB, dptrA, matrixDim);
+  checkCudaErrors(cudaMemcpyAsync(randValuesVerifyYmulX, dptrC, size,
+                                  cudaMemcpyDeviceToHost));
+  checkCudaErrors(cudaStreamSynchronize(NULL));
+#if VERIFY_GPU_CORRECTNESS
+  verifyMatrixMultiplyCorrectness(randValuesVerifyXmulY, randValuesX,
+                                  randValuesY, matrixDim);
+  verifyMatrixMultiplyCorrectness(randValuesVerifyYmulX, randValuesY,
+                                  randValuesX, matrixDim);
+#endif
+  checkCudaErrors(cudaFree(dptrA));
+  checkCudaErrors(cudaFree(dptrB));
+  checkCudaErrors(cudaFree(dptrC));
+
+  checkCudaErrors(cudaMallocHost(&latch, sizeof(unsigned int)));
+
+  switch (allocType) {
+    case USE_HOST_PAGEABLE_AND_DEVICE_MEMORY:
+    case USE_HOST_PAGEABLE_AND_DEVICE_MEMORY_ASYNC:
+      hptrA = (float *)malloc(size);
+      if (!hptrA) {
+        exit(EXIT_FAILURE);  // exit since memory allocation error
+      }
+      hptrB = (float *)malloc(size);
+      if (!hptrB) {
+        exit(EXIT_FAILURE);  // exit since memory allocation error
+      }
+      hptrC = (float *)malloc(size);
+      if (!hptrC) {
+        exit(EXIT_FAILURE);  // exit since memory allocation error
+      }
+      checkCudaErrors(cudaMalloc(&dptrA, size));
+      checkCudaErrors(cudaMalloc(&dptrB, size));
+      checkCudaErrors(cudaMalloc(&dptrC, size));
+      copyRequired = true;
+      break;
+
+    case USE_HOST_PAGELOCKED_AND_DEVICE_MEMORY:
+    case USE_HOST_PAGELOCKED_AND_DEVICE_MEMORY_ASYNC:
+      checkCudaErrors(cudaMallocHost(&hptrA, size));
+      checkCudaErrors(cudaMallocHost(&hptrB, size));
+      checkCudaErrors(cudaMallocHost(&hptrC, size));
+      checkCudaErrors(cudaMalloc(&dptrA, size));
+      checkCudaErrors(cudaMalloc(&dptrB, size));
+      checkCudaErrors(cudaMalloc(&dptrC, size));
+      copyRequired = true;
+      break;
+
+    case USE_ZERO_COPY:
+      checkCudaErrors(cudaMallocHost(&hptrA, size));
+      checkCudaErrors(cudaMallocHost(&hptrB, size));
+      checkCudaErrors(cudaMallocHost(&hptrC, size));
+      checkCudaErrors(cudaHostGetDevicePointer(&dptrA, hptrA, 0));
+      checkCudaErrors(cudaHostGetDevicePointer(&dptrB, hptrB, 0));
+      checkCudaErrors(cudaHostGetDevicePointer(&dptrC, hptrC, 0));
+      break;
+
+    case USE_MANAGED_MEMORY:
+      checkCudaErrors(cudaMallocManaged(&dptrA, size));
+      checkCudaErrors(cudaMallocManaged(&dptrB, size));
+      checkCudaErrors(cudaMallocManaged(&dptrC, size));
+      hptrA = dptrA;
+      hptrB = dptrB;
+      hptrC = dptrC;
+      break;
+
+    case USE_MANAGED_MEMORY_WITH_HINTS:
+    case USE_MANAGED_MEMORY_WITH_HINTS_ASYNC:
+      if (deviceProp.concurrentManagedAccess) {
+        checkCudaErrors(cudaMallocManaged(&dptrA, size));
+        checkCudaErrors(cudaMallocManaged(&dptrB, size));
+        checkCudaErrors(cudaMallocManaged(&dptrC, size));
+        checkCudaErrors(cudaMemPrefetchAsync(dptrA, size, cudaCpuDeviceId));
+        checkCudaErrors(cudaMemPrefetchAsync(dptrB, size, cudaCpuDeviceId));
+        checkCudaErrors(cudaMemPrefetchAsync(dptrC, size, cudaCpuDeviceId));
+      } else {
+        checkCudaErrors(cudaMallocManaged(&dptrA, size, cudaMemAttachHost));
+        checkCudaErrors(cudaMallocManaged(&dptrB, size, cudaMemAttachHost));
+        checkCudaErrors(cudaMallocManaged(&dptrC, size, cudaMemAttachHost));
+      }
+      hptrA = dptrA;
+      hptrB = dptrB;
+      hptrC = dptrC;
+      hintsRequired = true;
+      break;
+
+    default:
+      exit(EXIT_FAILURE);  // exit with error
+  }
+
+  if (allocType == USE_HOST_PAGEABLE_AND_DEVICE_MEMORY_ASYNC ||
+      allocType == USE_HOST_PAGELOCKED_AND_DEVICE_MEMORY_ASYNC ||
+      allocType == USE_MANAGED_MEMORY_WITH_HINTS_ASYNC) {
+    isAsync = true;
+  }
+
+  someTransferOpRequired = copyRequired || hintsRequired;
+
+  // fill buffers with 0 to avoid any first access page-fault overheads.
+  memset(hptrA, 0, size);
+  memset(hptrB, 0, size);
+  memset(hptrC, 0, size);
+
+  for (i = 0; i < numLoops; i++) {
+    cpuAccessTimes[i] = 0.0;
+    gpuLaunchCallsTimes[i] = 0.0;
+    gpuTransferToCallsTimes[i] = 0.0;
+    gpuTransferFromCallsTimes[i] = 0.0;
+
+    sdkStartTimer(&cpuAccessTimer);
+    {
+      copyMatrix(hptrA, (i & 0x1 == 0) ? randValuesX : randValuesY, matrixDim);
+      copyMatrix(hptrB, (i & 0x1 == 0) ? randValuesY : randValuesX, matrixDim);
+    }
+    sdkStopTimer(&cpuAccessTimer);
+    cpuAccessTimes[i] += sdkGetAverageTimerValue(&cpuAccessTimer);
+    sdkResetTimer(&cpuAccessTimer);
+
+    if (isAsync && hintsRequired) {
+      *latch = 0;
+      // Prevent any work on stream from starting until all work is pushed
+      spinWhileLessThanOne<<<1, 1, 0, streamToRunOn>>>(latch);
+    }
+
+    if (someTransferOpRequired) {
+      sdkStartTimer(&gpuTransferCallsTimer);
+      if (copyRequired) {
+        if (isAsync) {
+          checkCudaErrors(cudaMemcpyAsync(
+              dptrA, hptrA, size, cudaMemcpyHostToDevice, streamToRunOn));
+          checkCudaErrors(cudaMemcpyAsync(
+              dptrB, hptrB, size, cudaMemcpyHostToDevice, streamToRunOn));
+        } else {
+          checkCudaErrors(
+              cudaMemcpy(dptrA, hptrA, size, cudaMemcpyHostToDevice));
+          checkCudaErrors(
+              cudaMemcpy(dptrB, hptrB, size, cudaMemcpyHostToDevice));
+        }
+      }
+      if (hintsRequired) {
+        if (deviceProp.concurrentManagedAccess) {
+          checkCudaErrors(
+              cudaMemPrefetchAsync(dptrA, size, device_id, streamToRunOn));
+          checkCudaErrors(
+              cudaMemPrefetchAsync(dptrB, size, device_id, streamToRunOn));
+          checkCudaErrors(
+              cudaMemPrefetchAsync(dptrC, size, device_id, streamToRunOn));
+        } else {
+          checkCudaErrors(cudaStreamAttachMemAsync(streamToRunOn, dptrA, 0,
+                                                   cudaMemAttachGlobal));
+          checkCudaErrors(cudaStreamAttachMemAsync(streamToRunOn, dptrB, 0,
+                                                   cudaMemAttachGlobal));
+          checkCudaErrors(cudaStreamAttachMemAsync(streamToRunOn, dptrC, 0,
+                                                   cudaMemAttachGlobal));
+        }
+        if (!isAsync) {
+          checkCudaErrors(cudaStreamSynchronize(streamToRunOn));
+        }
+      }
+
+      sdkStopTimer(&gpuTransferCallsTimer);
+      gpuTransferToCallsTimes[i] +=
+          sdkGetAverageTimerValue(&gpuTransferCallsTimer);
+      sdkResetTimer(&gpuTransferCallsTimer);
+    }
+
+    sdkStartTimer(&gpuLaunchCallsTimer);
+    {
+      matrixMultiplyKernel<<<grid, threads, 0, streamToRunOn>>>(
+          dptrC, dptrA, dptrB, matrixDim);
+      if (!isAsync) {
+        checkCudaErrors(cudaStreamSynchronize(streamToRunOn));
+      }
+    }
+    sdkStopTimer(&gpuLaunchCallsTimer);
+
+    gpuLaunchCallsTimes[i] += sdkGetAverageTimerValue(&gpuLaunchCallsTimer);
+    sdkResetTimer(&gpuLaunchCallsTimer);
+
+    if (someTransferOpRequired) {
+      sdkStartTimer(&gpuTransferCallsTimer);
+      if (hintsRequired) {
+        if (deviceProp.concurrentManagedAccess) {
+          checkCudaErrors(cudaMemPrefetchAsync(dptrA, size, cudaCpuDeviceId));
+          checkCudaErrors(cudaMemPrefetchAsync(dptrB, size, cudaCpuDeviceId));
+          checkCudaErrors(cudaMemPrefetchAsync(dptrC, size, cudaCpuDeviceId));
+        } else {
+          checkCudaErrors(cudaStreamAttachMemAsync(streamToRunOn, dptrA, 0,
+                                                   cudaMemAttachHost));
+          checkCudaErrors(cudaStreamAttachMemAsync(streamToRunOn, dptrB, 0,
+                                                   cudaMemAttachHost));
+          checkCudaErrors(cudaStreamAttachMemAsync(streamToRunOn, dptrC, 0,
+                                                   cudaMemAttachHost));
+        }
+        if (!isAsync) {
+          checkCudaErrors(cudaStreamSynchronize(streamToRunOn));
+        }
+      }
+      if (copyRequired) {
+        if (isAsync) {
+          checkCudaErrors(cudaMemcpyAsync(
+              hptrC, dptrC, size, cudaMemcpyDeviceToHost, streamToRunOn));
+        } else {
+          checkCudaErrors(
+              cudaMemcpy(hptrC, dptrC, size, cudaMemcpyDeviceToHost));
+        }
+      }
+      sdkStopTimer(&gpuTransferCallsTimer);
+      gpuTransferFromCallsTimes[i] +=
+          sdkGetAverageTimerValue(&gpuTransferCallsTimer);
+      sdkResetTimer(&gpuTransferCallsTimer);
+    }
+    gpuLaunchAndTransferCallsTimes[i] = gpuLaunchCallsTimes[i] +
+                                        gpuTransferToCallsTimes[i] +
+                                        gpuTransferFromCallsTimes[i];
+    gpuLaunchTransferSyncTimes[i] = gpuLaunchAndTransferCallsTimes[i];
+    if (isAsync) {
+      sdkStartTimer(&gpuSyncTimer);
+      {
+        if (hintsRequired) {
+          *latch = 1;
+        }
+        checkCudaErrors(cudaStreamSynchronize(streamToRunOn));
+      }
+      sdkStopTimer(&gpuSyncTimer);
+      gpuLaunchTransferSyncTimes[i] += sdkGetAverageTimerValue(&gpuSyncTimer);
+      sdkResetTimer(&gpuSyncTimer);
+    }
+
+    sdkStartTimer(&cpuAccessTimer);
+    {
+      verifyMatrixData(
+          (i & 0x1 == 0) ? randValuesVerifyXmulY : randValuesVerifyYmulX, hptrC,
+          matrixDim);
+    }
+    sdkStopTimer(&cpuAccessTimer);
+    cpuAccessTimes[i] += sdkGetAverageTimerValue(&cpuAccessTimer);
+    sdkResetTimer(&cpuAccessTimer);
+    overallTimes[i] = cpuAccessTimes[i] + gpuLaunchTransferSyncTimes[i];
+  }
+
+  switch (allocType) {
+    case USE_HOST_PAGEABLE_AND_DEVICE_MEMORY:
+    case USE_HOST_PAGEABLE_AND_DEVICE_MEMORY_ASYNC:
+      free(hptrA);
+      free(hptrB);
+      free(hptrC);
+      checkCudaErrors(cudaFree(dptrA));
+      checkCudaErrors(cudaFree(dptrB));
+      checkCudaErrors(cudaFree(dptrC));
+      break;
+
+    case USE_HOST_PAGELOCKED_AND_DEVICE_MEMORY:
+    case USE_HOST_PAGELOCKED_AND_DEVICE_MEMORY_ASYNC:
+      checkCudaErrors(cudaFreeHost(hptrA));
+      checkCudaErrors(cudaFreeHost(hptrB));
+      checkCudaErrors(cudaFreeHost(hptrC));
+      checkCudaErrors(cudaFree(dptrA));
+      checkCudaErrors(cudaFree(dptrB));
+      checkCudaErrors(cudaFree(dptrC));
+      break;
+
+    case USE_ZERO_COPY:
+      checkCudaErrors(cudaFreeHost(hptrA));
+      checkCudaErrors(cudaFreeHost(hptrB));
+      checkCudaErrors(cudaFreeHost(hptrC));
+      break;
+
+    case USE_MANAGED_MEMORY:
+    case USE_MANAGED_MEMORY_WITH_HINTS:
+    case USE_MANAGED_MEMORY_WITH_HINTS_ASYNC:
+      checkCudaErrors(cudaFree(dptrA));
+      checkCudaErrors(cudaFree(dptrB));
+      checkCudaErrors(cudaFree(dptrC));
+      break;
+
+    default:
+      exit(EXIT_FAILURE);  // exit due to error
+  }
+
+  checkCudaErrors(cudaStreamDestroy(streamToRunOn));
+  checkCudaErrors(cudaFreeHost(latch));
+  free(randValuesX);
+  free(randValuesY);
+  free(randValuesVerifyXmulY);
+  free(randValuesVerifyYmulX);
+  sdkDeleteTimer(&gpuLaunchCallsTimer);
+  sdkDeleteTimer(&gpuTransferCallsTimer);
+  sdkDeleteTimer(&gpuSyncTimer);
+  sdkDeleteTimer(&cpuAccessTimer);
+}
+
+void matrixMultiplyPerfRunner(bool reportAsBandwidth,
+                              bool print_launch_transfer_results,
+                              bool print_std_deviation, int device_id) {
+  int i;
+  unsigned int minMatrixDim = 32;
+  unsigned int multiplierDim = 2;
+  unsigned int matrixDim;
+  unsigned int minSize = minMatrixDim * minMatrixDim * sizeof(float);
+  unsigned int maxSize =
+      (maxSampleSizeInMb * ONE_MB) /
+      4;  // 3 buffers are used, but dividing by 4 (power of 2)
+  unsigned int multiplier = multiplierDim * multiplierDim;
+  unsigned int numSizesToTest;
+
+  struct testResults *results;
+  struct resultsData *gpuLaunchCallsTimes;
+  struct resultsData *gpuTransferToCallsTimes;
+  struct resultsData *gpuTransferFromCallsTimes;
+  struct resultsData *gpuLaunchAndTransferCallsTimes;
+  struct resultsData *gpuLaunchTransferSyncTimes;
+  struct resultsData *cpuAccessTimes;
+  struct resultsData *overallTimes;
+  unsigned long *sizesToTest;
+  unsigned int j;
+
+  numSizesToTest = findNumSizesToTest(minSize, maxSize, multiplier);
+
+  createAndInitTestResults(&results, "matrixMultiplyPerf", numKernelRuns,
+                           numSizesToTest);
+
+  sizesToTest = getPtrSizesToTest(results);
+
+  createResultDataAndAddToTestResults(&gpuLaunchCallsTimes, results,
+                                      "GPU Kernel Launch Call Time", false,
+                                      reportAsBandwidth);
+  createResultDataAndAddToTestResults(&gpuTransferToCallsTimes, results,
+                                      "CPU to GPU Transfer Calls Time", false,
+                                      reportAsBandwidth);
+  createResultDataAndAddToTestResults(&gpuTransferFromCallsTimes, results,
+                                      "GPU to CPU Transfer Calls Time", false,
+                                      reportAsBandwidth);
+  createResultDataAndAddToTestResults(&gpuLaunchAndTransferCallsTimes, results,
+                                      "GPU Launch and Transfer Calls Time",
+                                      false, reportAsBandwidth);
+  createResultDataAndAddToTestResults(&gpuLaunchTransferSyncTimes, results,
+                                      "GPU Launch Transfer and Sync Time",
+                                      false, reportAsBandwidth);
+  createResultDataAndAddToTestResults(
+      &cpuAccessTimes, results, "CPU Access Time", false, reportAsBandwidth);
+  createResultDataAndAddToTestResults(&overallTimes, results, "Overall Time",
+                                      false, reportAsBandwidth);
+
+  printf("Running ");
+  for (matrixDim = minMatrixDim, j = 0;
+       matrixDim * matrixDim <= maxSize / sizeof(float);
+       matrixDim *= multiplierDim, ++j) {
+    sizesToTest[j] = matrixDim * matrixDim * sizeof(float);
+    for (i = MEMALLOC_TYPE_START; i <= MEMALLOC_TYPE_END; i++) {
+      printf(".");
+      fflush(stdout);
+      runMatrixMultiplyKernel(
+          matrixDim, i, numKernelRuns,
+          getPtrRunTimesInMs(gpuLaunchCallsTimes, i, j),
+          getPtrRunTimesInMs(gpuTransferToCallsTimes, i, j),
+          getPtrRunTimesInMs(gpuTransferFromCallsTimes, i, j),
+          getPtrRunTimesInMs(gpuLaunchAndTransferCallsTimes, i, j),
+          getPtrRunTimesInMs(gpuLaunchTransferSyncTimes, i, j),
+          getPtrRunTimesInMs(cpuAccessTimes, i, j),
+          getPtrRunTimesInMs(overallTimes, i, j), device_id);
+    }
+  }
+  printf("\n");
+  printResults(results, print_launch_transfer_results, print_std_deviation);
+  freeTestResultsAndAllResultsData(results);
+}
+
+static void usage() {
+  printf(
+      "./cudaMemoryTypesPerf [-device=<device_id>] [-reportAsBandwidth] "
+      "[-print-launch-transfer-results] [-print-std-deviation] [-verbose]\n");
+  printf("Options:\n");
+  printf(
+      "-reportAsBandwidth:             By default time taken is printed, this "
+      "option allows to instead print bandwidth.\n");
+  printf(
+      "-print-launch-transfer-results: By default overall results are printed, "
+      "this option allows to print data transfers and kernel time as well.\n");
+  printf(
+      "-print-std-deviation:           Prints std deviation of the results.\n");
+  printf(
+      "-kernel-iterations=<num>:       Number of times the kernel tests should "
+      "be run[default is 100 iterations].\n");
+  printf(
+      "-device=<device_id>:            Allows to pass GPU Device ID on which "
+      "the tests will be run.\n");
+  printf("-verbose:                       Prints highly verbose output.\n");
+}
+
+int main(int argc, char **argv) {
+  bool reportAsBandwidth = false;
+  bool print_launch_transfer_results = false;
+  bool print_std_deviation = false;
+
+  if (checkCmdLineFlag(argc, (const char **)argv, "help") ||
+      checkCmdLineFlag(argc, (const char **)argv, "h")) {
+    usage();
+    printf("&&&& %s WAIVED\n", argv[0]);
+    exit(EXIT_WAIVED);
+  }
+
+  if (checkCmdLineFlag(argc, (const char **)argv, "reportAsBandwidth")) {
+    reportAsBandwidth = true;
+  }
+
+  if (checkCmdLineFlag(argc, (const char **)argv,
+                       "print-launch-transfer-results")) {
+    print_launch_transfer_results = true;
+  }
+
+  if (checkCmdLineFlag(argc, (const char **)argv, "print-std-deviation")) {
+    print_std_deviation = true;
+  }
+
+  if (checkCmdLineFlag(argc, (const char **)argv, "kernel-iterations")) {
+    numKernelRuns =
+        getCmdLineArgumentInt(argc, (const char **)argv, "kernel-iterations");
+  }
+
+  if (checkCmdLineFlag(argc, (const char **)argv, "verbose")) {
+    verboseResults = 1;
+  }
+
+  int device_id = findCudaDevice(argc, (const char **)argv);
+
+  matrixMultiplyPerfRunner(reportAsBandwidth, print_launch_transfer_results,
+                           print_std_deviation, device_id);
+
+  printf(
+      "\nNOTE: The CUDA Samples are not meant for performance measurements. "
+      "Results may vary when GPU Boost is enabled.\n");
+  exit(EXIT_SUCCESS);
+}

Samples/6_Performance/alignedTypes/.vscode/c_cpp_properties.json

@@ -0,0 +1,18 @@
+{
+    "configurations": [
+        {
+            "name": "Linux",
+            "includePath": [
+                "${workspaceFolder}/**",
+                "${workspaceFolder}/../../../Common"
+            ],
+            "defines": [],
+            "compilerPath": "/usr/local/cuda/bin/nvcc",
+            "cStandard": "gnu17",
+            "cppStandard": "gnu++14",
+            "intelliSenseMode": "linux-gcc-x64",
+            "configurationProvider": "ms-vscode.makefile-tools"
+        }
+    ],
+    "version": 4
+}

Samples/6_Performance/alignedTypes/.vscode/extensions.json

@@ -0,0 +1,7 @@
+{
+    "recommendations": [
+        "nvidia.nsight-vscode-edition",
+        "ms-vscode.cpptools",
+        "ms-vscode.makefile-tools"
+    ]
+}

Samples/6_Performance/alignedTypes/.vscode/launch.json

@@ -0,0 +1,10 @@
+{
+    "configurations": [
+        {
+            "name": "CUDA C++: Launch",
+            "type": "cuda-gdb",
+            "request": "launch",
+            "program": "${workspaceFolder}/alignedTypes"
+        }
+    ]
+}

Samples/6_Performance/alignedTypes/.vscode/tasks.json

@@ -0,0 +1,15 @@
+{
+    "version": "2.0.0",
+    "tasks": [
+        {
+            "label": "sample",
+            "type": "shell",
+            "command": "make dbg=1",
+            "problemMatcher": ["$nvcc"],
+            "group": {
+                "kind": "build",
+                "isDefault": true
+            }
+        }
+    ]
+}

Samples/6_Performance/alignedTypes/Makefile

@@ -0,0 +1,340 @@
+################################################################################
+# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#  * Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#  * Neither the name of NVIDIA CORPORATION nor the names of its
+#    contributors may be used to endorse or promote products derived
+#    from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+################################################################################
+#
+# Makefile project only supported on Mac OS X and Linux Platforms)
+#
+################################################################################
+
+# Location of the CUDA Toolkit
+CUDA_PATH ?= /usr/local/cuda
+
+##############################
+# start deprecated interface #
+##############################
+ifeq ($(x86_64),1)
+    $(info WARNING - x86_64 variable has been deprecated)
+    $(info WARNING - please use TARGET_ARCH=x86_64 instead)
+    TARGET_ARCH ?= x86_64
+endif
+ifeq ($(ARMv7),1)
+    $(info WARNING - ARMv7 variable has been deprecated)
+    $(info WARNING - please use TARGET_ARCH=armv7l instead)
+    TARGET_ARCH ?= armv7l
+endif
+ifeq ($(aarch64),1)
+    $(info WARNING - aarch64 variable has been deprecated)
+    $(info WARNING - please use TARGET_ARCH=aarch64 instead)
+    TARGET_ARCH ?= aarch64
+endif
+ifeq ($(ppc64le),1)
+    $(info WARNING - ppc64le variable has been deprecated)
+    $(info WARNING - please use TARGET_ARCH=ppc64le instead)
+    TARGET_ARCH ?= ppc64le
+endif
+ifneq ($(GCC),)
+    $(info WARNING - GCC variable has been deprecated)
+    $(info WARNING - please use HOST_COMPILER=$(GCC) instead)
+    HOST_COMPILER ?= $(GCC)
+endif
+ifneq ($(abi),)
+    $(error ERROR - abi variable has been removed)
+endif
+############################
+# end deprecated interface #
+############################
+
+# architecture
+HOST_ARCH   := $(shell uname -m)
+TARGET_ARCH ?= $(HOST_ARCH)
+ifneq (,$(filter $(TARGET_ARCH),x86_64 aarch64 sbsa ppc64le armv7l))
+    ifneq ($(TARGET_ARCH),$(HOST_ARCH))
+        ifneq (,$(filter $(TARGET_ARCH),x86_64 aarch64 sbsa ppc64le))
+            TARGET_SIZE := 64
+        else ifneq (,$(filter $(TARGET_ARCH),armv7l))
+            TARGET_SIZE := 32
+        endif
+    else
+        TARGET_SIZE := $(shell getconf LONG_BIT)
+    endif
+else
+    $(error ERROR - unsupported value $(TARGET_ARCH) for TARGET_ARCH!)
+endif
+
+# sbsa and aarch64 systems look similar. Need to differentiate them at host level for now.
+ifeq ($(HOST_ARCH),aarch64)
+    ifeq ($(CUDA_PATH)/targets/sbsa-linux,$(shell ls -1d $(CUDA_PATH)/targets/sbsa-linux 2>/dev/null))
+        HOST_ARCH := sbsa
+        TARGET_ARCH := sbsa
+    endif
+endif
+
+ifneq ($(TARGET_ARCH),$(HOST_ARCH))
+    ifeq (,$(filter $(HOST_ARCH)-$(TARGET_ARCH),aarch64-armv7l x86_64-armv7l x86_64-aarch64 x86_64-sbsa x86_64-ppc64le))
+        $(error ERROR - cross compiling from $(HOST_ARCH) to $(TARGET_ARCH) is not supported!)
+    endif
+endif
+
+# When on native aarch64 system with userspace of 32-bit, change TARGET_ARCH to armv7l
+ifeq ($(HOST_ARCH)-$(TARGET_ARCH)-$(TARGET_SIZE),aarch64-aarch64-32)
+    TARGET_ARCH = armv7l
+endif
+
+# operating system
+HOST_OS   := $(shell uname -s 2>/dev/null | tr "[:upper:]" "[:lower:]")
+TARGET_OS ?= $(HOST_OS)
+ifeq (,$(filter $(TARGET_OS),linux darwin qnx android))
+    $(error ERROR - unsupported value $(TARGET_OS) for TARGET_OS!)
+endif
+
+# host compiler
+ifeq ($(TARGET_OS),darwin)
+    ifeq ($(shell expr `xcodebuild -version | grep -i xcode | awk '{print $$2}' | cut -d'.' -f1` \>= 5),1)
+        HOST_COMPILER ?= clang++
+    endif
+else ifneq ($(TARGET_ARCH),$(HOST_ARCH))
+    ifeq ($(HOST_ARCH)-$(TARGET_ARCH),x86_64-armv7l)
+        ifeq ($(TARGET_OS),linux)
+            HOST_COMPILER ?= arm-linux-gnueabihf-g++
+        else ifeq ($(TARGET_OS),qnx)
+            ifeq ($(QNX_HOST),)
+                $(error ERROR - QNX_HOST must be passed to the QNX host toolchain)
+            endif
+            ifeq ($(QNX_TARGET),)
+                $(error ERROR - QNX_TARGET must be passed to the QNX target toolchain)
+            endif
+            export QNX_HOST
+            export QNX_TARGET
+            HOST_COMPILER ?= $(QNX_HOST)/usr/bin/arm-unknown-nto-qnx6.6.0eabi-g++
+        else ifeq ($(TARGET_OS),android)
+            HOST_COMPILER ?= arm-linux-androideabi-g++
+        endif
+    else ifeq ($(TARGET_ARCH),aarch64)
+        ifeq ($(TARGET_OS), linux)
+            HOST_COMPILER ?= aarch64-linux-gnu-g++
+        else ifeq ($(TARGET_OS),qnx)
+            ifeq ($(QNX_HOST),)
+                $(error ERROR - QNX_HOST must be passed to the QNX host toolchain)
+            endif
+            ifeq ($(QNX_TARGET),)
+                $(error ERROR - QNX_TARGET must be passed to the QNX target toolchain)
+            endif
+            export QNX_HOST
+            export QNX_TARGET
+            HOST_COMPILER ?= $(QNX_HOST)/usr/bin/q++
+        else ifeq ($(TARGET_OS), android)
+            HOST_COMPILER ?= aarch64-linux-android-clang++
+        endif
+    else ifeq ($(TARGET_ARCH),sbsa)
+        HOST_COMPILER ?= aarch64-linux-gnu-g++
+    else ifeq ($(TARGET_ARCH),ppc64le)
+        HOST_COMPILER ?= powerpc64le-linux-gnu-g++
+    endif
+endif
+HOST_COMPILER ?= g++
+NVCC          := $(CUDA_PATH)/bin/nvcc -ccbin $(HOST_COMPILER)
+
+# internal flags
+NVCCFLAGS   := -m${TARGET_SIZE}
+CCFLAGS     :=
+LDFLAGS     :=
+
+# build flags
+ifeq ($(TARGET_OS),darwin)
+    LDFLAGS += -rpath $(CUDA_PATH)/lib
+    CCFLAGS += -arch $(HOST_ARCH)
+else ifeq ($(HOST_ARCH)-$(TARGET_ARCH)-$(TARGET_OS),x86_64-armv7l-linux)
+    LDFLAGS += --dynamic-linker=/lib/ld-linux-armhf.so.3
+    CCFLAGS += -mfloat-abi=hard
+else ifeq ($(TARGET_OS),android)
+    LDFLAGS += -pie
+    CCFLAGS += -fpie -fpic -fexceptions
+endif
+
+ifneq ($(TARGET_ARCH),$(HOST_ARCH))
+    ifeq ($(TARGET_ARCH)-$(TARGET_OS),armv7l-linux)
+        ifneq ($(TARGET_FS),)
+            GCCVERSIONLTEQ46 := $(shell expr `$(HOST_COMPILER) -dumpversion` \<= 4.6)
+            ifeq ($(GCCVERSIONLTEQ46),1)
+                CCFLAGS += --sysroot=$(TARGET_FS)
+            endif
+            LDFLAGS += --sysroot=$(TARGET_FS)
+            LDFLAGS += -rpath-link=$(TARGET_FS)/lib
+            LDFLAGS += -rpath-link=$(TARGET_FS)/usr/lib
+            LDFLAGS += -rpath-link=$(TARGET_FS)/usr/lib/arm-linux-gnueabihf
+        endif
+    endif
+    ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-linux)
+        ifneq ($(TARGET_FS),)
+            GCCVERSIONLTEQ46 := $(shell expr `$(HOST_COMPILER) -dumpversion` \<= 4.6)
+            ifeq ($(GCCVERSIONLTEQ46),1)
+                CCFLAGS += --sysroot=$(TARGET_FS)
+            endif
+            LDFLAGS += --sysroot=$(TARGET_FS)
+            LDFLAGS += -rpath-link=$(TARGET_FS)/lib -L$(TARGET_FS)/lib
+            LDFLAGS += -rpath-link=$(TARGET_FS)/lib/aarch64-linux-gnu -L$(TARGET_FS)/lib/aarch64-linux-gnu
+            LDFLAGS += -rpath-link=$(TARGET_FS)/usr/lib -L$(TARGET_FS)/usr/lib
+            LDFLAGS += -rpath-link=$(TARGET_FS)/usr/lib/aarch64-linux-gnu -L$(TARGET_FS)/usr/lib/aarch64-linux-gnu
+            LDFLAGS += --unresolved-symbols=ignore-in-shared-libs
+            CCFLAGS += -isystem=$(TARGET_FS)/usr/include -I$(TARGET_FS)/usr/include -I$(TARGET_FS)/usr/include/libdrm
+            CCFLAGS += -isystem=$(TARGET_FS)/usr/include/aarch64-linux-gnu -I$(TARGET_FS)/usr/include/aarch64-linux-gnu
+        endif
+    endif
+    ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-qnx)
+        NVCCFLAGS += -D_QNX_SOURCE
+        NVCCFLAGS += --qpp-config 8.3.0,gcc_ntoaarch64le
+        CCFLAGS += -DWIN_INTERFACE_CUSTOM -I/usr/include/aarch64-qnx-gnu
+        LDFLAGS += -lsocket
+        LDFLAGS += -L/usr/lib/aarch64-qnx-gnu
+        CCFLAGS += "-Wl\,-rpath-link\,/usr/lib/aarch64-qnx-gnu"
+        ifdef TARGET_OVERRIDE
+            LDFLAGS += -lslog2
+        endif
+
+        ifneq ($(TARGET_FS),)
+            LDFLAGS += -L$(TARGET_FS)/usr/lib
+            CCFLAGS += "-Wl\,-rpath-link\,$(TARGET_FS)/usr/lib"
+            LDFLAGS += -L$(TARGET_FS)/usr/libnvidia
+            CCFLAGS += "-Wl\,-rpath-link\,$(TARGET_FS)/usr/libnvidia"
+            CCFLAGS += -I$(TARGET_FS)/../include
+        endif
+    endif
+endif
+
+ifdef TARGET_OVERRIDE # cuda toolkit targets override
+    NVCCFLAGS += -target-dir $(TARGET_OVERRIDE)
+endif
+
+# Install directory of different arch
+CUDA_INSTALL_TARGET_DIR :=
+ifeq ($(TARGET_ARCH)-$(TARGET_OS),armv7l-linux)
+    CUDA_INSTALL_TARGET_DIR = targets/armv7-linux-gnueabihf/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-linux)
+    CUDA_INSTALL_TARGET_DIR = targets/aarch64-linux/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),sbsa-linux)
+    CUDA_INSTALL_TARGET_DIR = targets/sbsa-linux/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),armv7l-android)
+    CUDA_INSTALL_TARGET_DIR = targets/armv7-linux-androideabi/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-android)
+    CUDA_INSTALL_TARGET_DIR = targets/aarch64-linux-androideabi/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),armv7l-qnx)
+    CUDA_INSTALL_TARGET_DIR = targets/ARMv7-linux-QNX/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-qnx)
+    CUDA_INSTALL_TARGET_DIR = targets/aarch64-qnx/
+else ifeq ($(TARGET_ARCH),ppc64le)
+    CUDA_INSTALL_TARGET_DIR = targets/ppc64le-linux/
+endif
+
+# Debug build flags
+ifeq ($(dbg),1)
+      NVCCFLAGS += -g -G
+      BUILD_TYPE := debug
+else
+      BUILD_TYPE := release
+endif
+
+ALL_CCFLAGS :=
+ALL_CCFLAGS += $(NVCCFLAGS)
+ALL_CCFLAGS += $(EXTRA_NVCCFLAGS)
+ALL_CCFLAGS += $(addprefix -Xcompiler ,$(CCFLAGS))
+ALL_CCFLAGS += $(addprefix -Xcompiler ,$(EXTRA_CCFLAGS))
+
+SAMPLE_ENABLED := 1
+
+ALL_LDFLAGS :=
+ALL_LDFLAGS += $(ALL_CCFLAGS)
+ALL_LDFLAGS += $(addprefix -Xlinker ,$(LDFLAGS))
+ALL_LDFLAGS += $(addprefix -Xlinker ,$(EXTRA_LDFLAGS))
+
+# Common includes and paths for CUDA
+INCLUDES  := -I../../../Common
+LIBRARIES :=
+
+################################################################################
+
+# Gencode arguments
+ifeq ($(TARGET_ARCH),$(filter $(TARGET_ARCH),armv7l aarch64 sbsa))
+SMS ?= 53 61 70 72 75 80 86 87
+else
+SMS ?= 35 37 50 52 60 61 70 75 80 86
+endif
+
+ifeq ($(SMS),)
+$(info >>> WARNING - no SM architectures have been specified - waiving sample <<<)
+SAMPLE_ENABLED := 0
+endif
+
+ifeq ($(GENCODE_FLAGS),)
+# Generate SASS code for each SM architecture listed in $(SMS)
+$(foreach sm,$(SMS),$(eval GENCODE_FLAGS += -gencode arch=compute_$(sm),code=sm_$(sm)))
+
+# Generate PTX code from the highest SM architecture in $(SMS) to guarantee forward-compatibility
+HIGHEST_SM := $(lastword $(sort $(SMS)))
+ifneq ($(HIGHEST_SM),)
+GENCODE_FLAGS += -gencode arch=compute_$(HIGHEST_SM),code=compute_$(HIGHEST_SM)
+endif
+endif
+
+ALL_CCFLAGS += --threads 0 --std=c++11
+
+ifeq ($(SAMPLE_ENABLED),0)
+EXEC ?= @echo "[@]"
+endif
+
+################################################################################
+
+# Target rules
+all: build
+
+build: alignedTypes
+
+check.deps:
+ifeq ($(SAMPLE_ENABLED),0)
+	@echo "Sample will be waived due to the above missing dependencies"
+else
+	@echo "Sample is ready - all dependencies have been met"
+endif
+
+alignedTypes.o:alignedTypes.cu
+	$(EXEC) $(NVCC) $(INCLUDES) $(ALL_CCFLAGS) $(GENCODE_FLAGS) -o $@ -c $<
+
+alignedTypes: alignedTypes.o
+	$(EXEC) $(NVCC) $(ALL_LDFLAGS) $(GENCODE_FLAGS) -o $@ $+ $(LIBRARIES)
+	$(EXEC) mkdir -p ../../../bin/$(TARGET_ARCH)/$(TARGET_OS)/$(BUILD_TYPE)
+	$(EXEC) cp $@ ../../../bin/$(TARGET_ARCH)/$(TARGET_OS)/$(BUILD_TYPE)
+
+run: build
+	$(EXEC) ./alignedTypes
+
+testrun: build
+
+clean:
+	rm -f alignedTypes alignedTypes.o
+	rm -rf ../../../bin/$(TARGET_ARCH)/$(TARGET_OS)/$(BUILD_TYPE)/alignedTypes
+
+clobber: clean

Samples/6_Performance/alignedTypes/NsightEclipse.xml

@@ -0,0 +1,77 @@
+<?xml version="1.0" encoding="UTF-8"?> 
+<!DOCTYPE entry SYSTEM "SamplesInfo.dtd">
+<entry>
+  <name>alignedTypes</name>
+  <cuda_api_list>
+    <toolkit>cudaMemset</toolkit>
+    <toolkit>cudaFree</toolkit>
+    <toolkit>cudaDeviceSynchronize</toolkit>
+    <toolkit>cudaMalloc</toolkit>
+    <toolkit>cudaMemcpy</toolkit>
+    <toolkit>cudaGetDeviceProperties</toolkit>
+  </cuda_api_list>
+  <description><![CDATA[A simple test, showing huge access speed gap between aligned and misaligned structures. It measures per-element copy throughput for aligned and misaligned structures on big chunks of data.]]></description>
+  <devicecompilation>whole</devicecompilation>
+  <includepaths>
+    <path>./</path>
+    <path>../</path>
+    <path>../../../Common</path>
+  </includepaths>
+  <keyconcepts>
+    <concept level="advanced">Performance Strategies</concept>
+  </keyconcepts>
+  <keywords>
+    <keyword>CUDA</keyword>
+  </keywords>
+  <libraries>
+  </libraries>
+  <librarypaths>
+  </librarypaths>
+  <nsight_eclipse>true</nsight_eclipse>
+  <primary_file>alignedTypes.cu</primary_file>
+  <scopes>
+    <scope>1:CUDA Basic Topics</scope>
+    <scope>1:Performance Strategies</scope>
+  </scopes>
+  <sm-arch>sm35</sm-arch>
+  <sm-arch>sm37</sm-arch>
+  <sm-arch>sm50</sm-arch>
+  <sm-arch>sm52</sm-arch>
+  <sm-arch>sm53</sm-arch>
+  <sm-arch>sm60</sm-arch>
+  <sm-arch>sm61</sm-arch>
+  <sm-arch>sm70</sm-arch>
+  <sm-arch>sm72</sm-arch>
+  <sm-arch>sm75</sm-arch>
+  <sm-arch>sm80</sm-arch>
+  <sm-arch>sm86</sm-arch>
+  <sm-arch>sm87</sm-arch>
+  <supported_envs>
+    <env>
+      <arch>x86_64</arch>
+      <platform>linux</platform>
+    </env>
+    <env>
+      <platform>windows7</platform>
+    </env>
+    <env>
+      <arch>x86_64</arch>
+      <platform>macosx</platform>
+    </env>
+    <env>
+      <arch>arm</arch>
+    </env>
+    <env>
+      <arch>sbsa</arch>
+    </env>
+    <env>
+      <arch>ppc64le</arch>
+      <platform>linux</platform>
+    </env>
+  </supported_envs>
+  <supported_sm_architectures>
+    <include>all</include>
+  </supported_sm_architectures>
+  <title>Aligned Types</title>
+  <type>exe</type>
+</entry>

Samples/6_Performance/alignedTypes/README.md

@@ -0,0 +1,70 @@
+# alignedTypes - Aligned Types
+
+## Description
+
+A simple test, showing huge access speed gap between aligned and misaligned structures. It measures per-element copy throughput for aligned and misaligned structures on big chunks of data.
+
+## Key Concepts
+
+Performance Strategies
+
+## Supported SM Architectures
+
+[SM 3.5 ](https://developer.nvidia.com/cuda-gpus)  [SM 3.7 ](https://developer.nvidia.com/cuda-gpus)  [SM 5.0 ](https://developer.nvidia.com/cuda-gpus)  [SM 5.2 ](https://developer.nvidia.com/cuda-gpus)  [SM 5.3 ](https://developer.nvidia.com/cuda-gpus)  [SM 6.0 ](https://developer.nvidia.com/cuda-gpus)  [SM 6.1 ](https://developer.nvidia.com/cuda-gpus)  [SM 7.0 ](https://developer.nvidia.com/cuda-gpus)  [SM 7.2 ](https://developer.nvidia.com/cuda-gpus)  [SM 7.5 ](https://developer.nvidia.com/cuda-gpus)  [SM 8.0 ](https://developer.nvidia.com/cuda-gpus)  [SM 8.6 ](https://developer.nvidia.com/cuda-gpus)  [SM 8.7 ](https://developer.nvidia.com/cuda-gpus)
+
+## Supported OSes
+
+Linux, Windows
+
+## Supported CPU Architecture
+
+x86_64, ppc64le, armv7l
+
+## CUDA APIs involved
+
+### [CUDA Runtime API](http://docs.nvidia.com/cuda/cuda-runtime-api/index.html)
+cudaMemset, cudaFree, cudaDeviceSynchronize, cudaMalloc, cudaMemcpy, cudaGetDeviceProperties
+
+## Prerequisites
+
+Download and install the [CUDA Toolkit 11.6](https://developer.nvidia.com/cuda-downloads) for your corresponding platform.
+
+## Build and Run
+
+### Windows
+The Windows samples are built using the Visual Studio IDE. Solution files (.sln) are provided for each supported version of Visual Studio, using the format:
+```
+*_vs<version>.sln - for Visual Studio <version>
+```
+Each individual sample has its own set of solution files in its directory:
+
+To build/examine all the samples at once, the complete solution files should be used. To build/examine a single sample, the individual sample solution files should be used.
+> **Note:** Some samples require that the Microsoft DirectX SDK (June 2010 or newer) be installed and that the VC++ directory paths are properly set up (**Tools > Options...**). Check DirectX Dependencies section for details."
+
+### Linux
+The Linux samples are built using makefiles. To use the makefiles, change the current directory to the sample directory you wish to build, and run make:
+```
+$ cd <sample_dir>
+$ make
+```
+The samples makefiles can take advantage of certain options:
+*  **TARGET_ARCH=<arch>** - cross-compile targeting a specific architecture. Allowed architectures are x86_64, ppc64le, armv7l.
+    By default, TARGET_ARCH is set to HOST_ARCH. On a x86_64 machine, not setting TARGET_ARCH is the equivalent of setting TARGET_ARCH=x86_64.<br/>
+`$ make TARGET_ARCH=x86_64` <br/> `$ make TARGET_ARCH=ppc64le` <br/> `$ make TARGET_ARCH=armv7l` <br/>
+    See [here](http://docs.nvidia.com/cuda/cuda-samples/index.html#cross-samples) for more details.
+*   **dbg=1** - build with debug symbols
+    ```
+    $ make dbg=1
+    ```
+*   **SMS="A B ..."** - override the SM architectures for which the sample will be built, where `"A B ..."` is a space-delimited list of SM architectures. For example, to generate SASS for SM 50 and SM 60, use `SMS="50 60"`.
+    ```
+    $ make SMS="50 60"
+    ```
+
+*  **HOST_COMPILER=<host_compiler>** - override the default g++ host compiler. See the [Linux Installation Guide](http://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html#system-requirements) for a list of supported host compilers.
+```
+    $ make HOST_COMPILER=g++
+```
+
+## References (for more details)
+

Samples/6_Performance/alignedTypes/alignedTypes.cu

@@ -0,0 +1,314 @@
+/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * This is a simple test showing huge access speed gap
+ * between aligned and misaligned structures
+ * (those having/missing __align__ keyword).
+ * It measures per-element copy throughput for
+ * aligned and misaligned structures on
+ * big chunks of data.
+ */
+
+// includes, system
+#include <math.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+// includes, project
+#include <helper_cuda.h>  // helper functions for CUDA error checking and initialization
+#include <helper_functions.h>  // helper utility functions
+
+////////////////////////////////////////////////////////////////////////////////
+// Misaligned types
+////////////////////////////////////////////////////////////////////////////////
+typedef unsigned char uint8;
+
+typedef unsigned short int uint16;
+
+typedef struct {
+  unsigned char r, g, b, a;
+} RGBA8_misaligned;
+
+typedef struct {
+  unsigned int l, a;
+} LA32_misaligned;
+
+typedef struct {
+  unsigned int r, g, b;
+} RGB32_misaligned;
+
+typedef struct {
+  unsigned int r, g, b, a;
+} RGBA32_misaligned;
+
+////////////////////////////////////////////////////////////////////////////////
+// Aligned types
+////////////////////////////////////////////////////////////////////////////////
+typedef struct __align__(4) {
+  unsigned char r, g, b, a;
+}
+RGBA8;
+
+typedef unsigned int I32;
+
+typedef struct __align__(8) {
+  unsigned int l, a;
+}
+LA32;
+
+typedef struct __align__(16) {
+  unsigned int r, g, b;
+}
+RGB32;
+
+typedef struct __align__(16) {
+  unsigned int r, g, b, a;
+}
+RGBA32;
+
+////////////////////////////////////////////////////////////////////////////////
+// Because G80 class hardware natively supports global memory operations
+// only with data elements of 4, 8 and 16 bytes, if structure size
+// exceeds 16 bytes, it can't be efficiently read or written,
+// since more than one global memory non-coalescable load/store instructions
+// will be generated, even if __align__ option is supplied.
+// "Structure of arrays" storage strategy offers best performance
+// in general case. See section 5.1.2 of the Programming Guide.
+////////////////////////////////////////////////////////////////////////////////
+typedef struct __align__(16) {
+  RGBA32 c1, c2;
+}
+RGBA32_2;
+
+////////////////////////////////////////////////////////////////////////////////
+// Common host and device functions
+////////////////////////////////////////////////////////////////////////////////
+// Round a / b to nearest higher integer value
+int iDivUp(int a, int b) { return (a % b != 0) ? (a / b + 1) : (a / b); }
+
+// Round a / b to nearest lower integer value
+int iDivDown(int a, int b) { return a / b; }
+
+// Align a to nearest higher multiple of b
+int iAlignUp(int a, int b) { return (a % b != 0) ? (a - a % b + b) : a; }
+
+// Align a to nearest lower multiple of b
+int iAlignDown(int a, int b) { return a - a % b; }
+
+////////////////////////////////////////////////////////////////////////////////
+// Simple CUDA kernel.
+// Copy is carried out on per-element basis,
+// so it's not per-byte in case of padded structures.
+////////////////////////////////////////////////////////////////////////////////
+template <class TData>
+__global__ void testKernel(TData *d_odata, TData *d_idata, int numElements) {
+  const int tid = blockDim.x * blockIdx.x + threadIdx.x;
+  const int numThreads = blockDim.x * gridDim.x;
+
+  for (int pos = tid; pos < numElements; pos += numThreads) {
+    d_odata[pos] = d_idata[pos];
+  }
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// Validation routine for simple copy kernel.
+// We must know "packed" size of TData (number_of_fields * sizeof(simple_type))
+// and compare only these "packed" parts of the structure,
+// containing actual user data. The compiler behavior with padding bytes
+// is undefined, since padding is merely a placeholder
+// and doesn't contain any user data.
+////////////////////////////////////////////////////////////////////////////////
+template <class TData>
+int testCPU(TData *h_odata, TData *h_idata, int numElements,
+            int packedElementSize) {
+  for (int pos = 0; pos < numElements; pos++) {
+    TData src = h_idata[pos];
+    TData dst = h_odata[pos];
+
+    for (int i = 0; i < packedElementSize; i++)
+      if (((char *)&src)[i] != ((char *)&dst)[i]) {
+        return 0;
+      }
+  }
+
+  return 1;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// Data configuration
+////////////////////////////////////////////////////////////////////////////////
+// Memory chunk size in bytes. Reused for test
+const int MEM_SIZE = 50000000;
+const int NUM_ITERATIONS = 32;
+
+// GPU input and output data
+unsigned char *d_idata, *d_odata;
+// CPU input data and instance of GPU output data
+unsigned char *h_idataCPU, *h_odataGPU;
+StopWatchInterface *hTimer = NULL;
+
+template <class TData>
+int runTest(int packedElementSize, int memory_size) {
+  const int totalMemSizeAligned = iAlignDown(memory_size, sizeof(TData));
+  const int numElements = iDivDown(memory_size, sizeof(TData));
+
+  // Clean output buffer before current test
+  checkCudaErrors(cudaMemset(d_odata, 0, memory_size));
+  // Run test
+  checkCudaErrors(cudaDeviceSynchronize());
+  sdkResetTimer(&hTimer);
+  sdkStartTimer(&hTimer);
+
+  for (int i = 0; i < NUM_ITERATIONS; i++) {
+    testKernel<TData>
+        <<<64, 256>>>((TData *)d_odata, (TData *)d_idata, numElements);
+    getLastCudaError("testKernel() execution failed\n");
+  }
+
+  checkCudaErrors(cudaDeviceSynchronize());
+  sdkStopTimer(&hTimer);
+  double gpuTime = sdkGetTimerValue(&hTimer) / NUM_ITERATIONS;
+  printf("Avg. time: %f ms / Copy throughput: %f GB/s.\n", gpuTime,
+         (double)totalMemSizeAligned / (gpuTime * 0.001 * 1073741824.0));
+
+  // Read back GPU results and run validation
+  checkCudaErrors(
+      cudaMemcpy(h_odataGPU, d_odata, memory_size, cudaMemcpyDeviceToHost));
+  int flag = testCPU((TData *)h_odataGPU, (TData *)h_idataCPU, numElements,
+                     packedElementSize);
+
+  printf(flag ? "\tTEST OK\n" : "\tTEST FAILURE\n");
+
+  return !flag;
+}
+
+int main(int argc, char **argv) {
+  int i, nTotalFailures = 0;
+
+  int devID;
+  cudaDeviceProp deviceProp;
+  printf("[%s] - Starting...\n", argv[0]);
+
+  // find first CUDA device
+  devID = findCudaDevice(argc, (const char **)argv);
+
+  // get number of SMs on this GPU
+  checkCudaErrors(cudaGetDeviceProperties(&deviceProp, devID));
+  printf("[%s] has %d MP(s) x %d (Cores/MP) = %d (Cores)\n", deviceProp.name,
+         deviceProp.multiProcessorCount,
+         _ConvertSMVer2Cores(deviceProp.major, deviceProp.minor),
+         _ConvertSMVer2Cores(deviceProp.major, deviceProp.minor) *
+             deviceProp.multiProcessorCount);
+
+  // Anything that is less than 192 Cores will have a scaled down workload
+  float scale_factor =
+      max((192.0f / (_ConvertSMVer2Cores(deviceProp.major, deviceProp.minor) *
+                     (float)deviceProp.multiProcessorCount)),
+          1.0f);
+
+  int MemorySize = (int)(MEM_SIZE / scale_factor) &
+                   0xffffff00;  // force multiple of 256 bytes
+
+  printf("> Compute scaling value = %4.2f\n", scale_factor);
+  printf("> Memory Size = %d\n", MemorySize);
+
+  sdkCreateTimer(&hTimer);
+
+  printf("Allocating memory...\n");
+  h_idataCPU = (unsigned char *)malloc(MemorySize);
+  h_odataGPU = (unsigned char *)malloc(MemorySize);
+  checkCudaErrors(cudaMalloc((void **)&d_idata, MemorySize));
+  checkCudaErrors(cudaMalloc((void **)&d_odata, MemorySize));
+
+  printf("Generating host input data array...\n");
+
+  for (i = 0; i < MemorySize; i++) {
+    h_idataCPU[i] = (i & 0xFF) + 1;
+  }
+
+  printf("Uploading input data to GPU memory...\n");
+  checkCudaErrors(
+      cudaMemcpy(d_idata, h_idataCPU, MemorySize, cudaMemcpyHostToDevice));
+
+  printf("Testing misaligned types...\n");
+  printf("uint8...\n");
+  nTotalFailures += runTest<uint8>(1, MemorySize);
+
+  printf("uint16...\n");
+  nTotalFailures += runTest<uint16>(2, MemorySize);
+
+  printf("RGBA8_misaligned...\n");
+  nTotalFailures += runTest<RGBA8_misaligned>(4, MemorySize);
+
+  printf("LA32_misaligned...\n");
+  nTotalFailures += runTest<LA32_misaligned>(8, MemorySize);
+
+  printf("RGB32_misaligned...\n");
+  nTotalFailures += runTest<RGB32_misaligned>(12, MemorySize);
+
+  printf("RGBA32_misaligned...\n");
+  nTotalFailures += runTest<RGBA32_misaligned>(16, MemorySize);
+
+  printf("Testing aligned types...\n");
+  printf("RGBA8...\n");
+  nTotalFailures += runTest<RGBA8>(4, MemorySize);
+
+  printf("I32...\n");
+  nTotalFailures += runTest<I32>(4, MemorySize);
+
+  printf("LA32...\n");
+  nTotalFailures += runTest<LA32>(8, MemorySize);
+
+  printf("RGB32...\n");
+  nTotalFailures += runTest<RGB32>(12, MemorySize);
+
+  printf("RGBA32...\n");
+  nTotalFailures += runTest<RGBA32>(16, MemorySize);
+
+  printf("RGBA32_2...\n");
+  nTotalFailures += runTest<RGBA32_2>(32, MemorySize);
+
+  printf("\n[alignedTypes] -> Test Results: %d Failures\n", nTotalFailures);
+
+  printf("Shutting down...\n");
+  checkCudaErrors(cudaFree(d_idata));
+  checkCudaErrors(cudaFree(d_odata));
+  free(h_odataGPU);
+  free(h_idataCPU);
+
+  sdkDeleteTimer(&hTimer);
+
+  if (nTotalFailures != 0) {
+    printf("Test failed!\n");
+    exit(EXIT_FAILURE);
+  }
+
+  printf("Test passed\n");
+  exit(EXIT_SUCCESS);
+}

Samples/6_Performance/alignedTypes/alignedTypes_vs2017.sln

@@ -0,0 +1,20 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio 2017
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "alignedTypes", "alignedTypes_vs2017.vcxproj", "{997E0757-EA74-4A4E-A0FC-47D8C8831A15}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|x64 = Debug|x64
+		Release|x64 = Release|x64
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Debug|x64.ActiveCfg = Debug|x64
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Debug|x64.Build.0 = Debug|x64
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Release|x64.ActiveCfg = Release|x64
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Release|x64.Build.0 = Release|x64
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+EndGlobal

Samples/6_Performance/alignedTypes/alignedTypes_vs2017.vcxproj

@@ -0,0 +1,112 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <CUDAPropsPath Condition="'$(CUDAPropsPath)'==''">$(VCTargetsPath)\BuildCustomizations</CUDAPropsPath>
+  </PropertyGroup>
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Debug|x64">
+      <Configuration>Debug</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|x64">
+      <Configuration>Release</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>{997E0757-EA74-4A4E-A0FC-47D8C8831A15}</ProjectGuid>
+    <RootNamespace>alignedTypes_vs2017</RootNamespace>
+    <ProjectName>alignedTypes</ProjectName>
+    <CudaToolkitCustomDir />
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(WindowsTargetPlatformVersion)'==''">
+    <LatestTargetPlatformVersion>$([Microsoft.Build.Utilities.ToolLocationHelper]::GetLatestSDKTargetPlatformVersion('Windows', '10.0'))</LatestTargetPlatformVersion>
+    <WindowsTargetPlatformVersion Condition="'$(WindowsTargetPlatformVersion)' == ''">$(LatestTargetPlatformVersion)</WindowsTargetPlatformVersion>
+    <TargetPlatformVersion>$(WindowsTargetPlatformVersion)</TargetPlatformVersion>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <PropertyGroup>
+    <ConfigurationType>Application</ConfigurationType>
+    <CharacterSet>MultiByte</CharacterSet>
+    <PlatformToolset>v141</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)'=='Debug'">
+    <UseDebugLibraries>true</UseDebugLibraries>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)'=='Release'">
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <ImportGroup Label="ExtensionSettings">
+    <Import Project="$(CUDAPropsPath)\CUDA 11.6.props" />
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets">
+    <Import Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" />
+  </ImportGroup>
+  <PropertyGroup Label="UserMacros" />
+  <PropertyGroup>
+    <IntDir>$(Platform)/$(Configuration)/</IntDir>
+    <IncludePath>$(IncludePath)</IncludePath>
+    <CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRules />
+    <CodeAnalysisRuleAssemblies />
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Platform)'=='x64'">
+    <OutDir>../../../bin/win64/$(Configuration)/</OutDir>
+  </PropertyGroup>
+  <ItemDefinitionGroup>
+    <ClCompile>
+      <WarningLevel>Level3</WarningLevel>
+      <PreprocessorDefinitions>WIN32;_MBCS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <AdditionalIncludeDirectories>./;$(CudaToolkitDir)/include;../../../Common;</AdditionalIncludeDirectories>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <AdditionalDependencies>cudart_static.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalLibraryDirectories>$(CudaToolkitLibDir);</AdditionalLibraryDirectories>
+      <OutputFile>$(OutDir)/alignedTypes.exe</OutputFile>
+    </Link>
+    <CudaCompile>
+      <CodeGeneration>compute_35,sm_35;compute_37,sm_37;compute_50,sm_50;compute_52,sm_52;compute_60,sm_60;compute_61,sm_61;compute_70,sm_70;compute_75,sm_75;compute_80,sm_80;compute_86,sm_86;</CodeGeneration>
+      <AdditionalOptions>-Xcompiler "/wd 4819"  --threads 0 </AdditionalOptions>
+      <Include>./;../../../Common</Include>
+      <Defines>WIN32</Defines>
+    </CudaCompile>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)'=='Debug'">
+    <ClCompile>
+      <Optimization>Disabled</Optimization>
+      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <LinkTimeCodeGeneration>Default</LinkTimeCodeGeneration>
+    </Link>
+    <CudaCompile>
+      <Runtime>MTd</Runtime>
+      <TargetMachinePlatform>64</TargetMachinePlatform>
+    </CudaCompile>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)'=='Release'">
+    <ClCompile>
+      <Optimization>MaxSpeed</Optimization>
+      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <GenerateDebugInformation>false</GenerateDebugInformation>
+      <LinkTimeCodeGeneration>UseLinkTimeCodeGeneration</LinkTimeCodeGeneration>
+    </Link>
+    <CudaCompile>
+      <Runtime>MT</Runtime>
+      <TargetMachinePlatform>64</TargetMachinePlatform>
+    </CudaCompile>
+  </ItemDefinitionGroup>
+  <ItemGroup>
+    <CudaCompile Include="alignedTypes.cu" />
+
+  </ItemGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <ImportGroup Label="ExtensionTargets">
+    <Import Project="$(CUDAPropsPath)\CUDA 11.6.targets" />
+  </ImportGroup>
+</Project>

Samples/6_Performance/alignedTypes/alignedTypes_vs2019.sln

@@ -0,0 +1,20 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio 2019
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "alignedTypes", "alignedTypes_vs2019.vcxproj", "{997E0757-EA74-4A4E-A0FC-47D8C8831A15}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|x64 = Debug|x64
+		Release|x64 = Release|x64
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Debug|x64.ActiveCfg = Debug|x64
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Debug|x64.Build.0 = Debug|x64
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Release|x64.ActiveCfg = Release|x64
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Release|x64.Build.0 = Release|x64
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+EndGlobal

Samples/6_Performance/alignedTypes/alignedTypes_vs2019.vcxproj

@@ -0,0 +1,108 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <CUDAPropsPath Condition="'$(CUDAPropsPath)'==''">$(VCTargetsPath)\BuildCustomizations</CUDAPropsPath>
+  </PropertyGroup>
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Debug|x64">
+      <Configuration>Debug</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|x64">
+      <Configuration>Release</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>{997E0757-EA74-4A4E-A0FC-47D8C8831A15}</ProjectGuid>
+    <RootNamespace>alignedTypes_vs2019</RootNamespace>
+    <ProjectName>alignedTypes</ProjectName>
+    <CudaToolkitCustomDir />
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <PropertyGroup>
+    <ConfigurationType>Application</ConfigurationType>
+    <CharacterSet>MultiByte</CharacterSet>
+    <PlatformToolset>v142</PlatformToolset>
+    <WindowsTargetPlatformVersion>10.0</WindowsTargetPlatformVersion>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)'=='Debug'">
+    <UseDebugLibraries>true</UseDebugLibraries>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)'=='Release'">
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <ImportGroup Label="ExtensionSettings">
+    <Import Project="$(CUDAPropsPath)\CUDA 11.6.props" />
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets">
+    <Import Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" />
+  </ImportGroup>
+  <PropertyGroup Label="UserMacros" />
+  <PropertyGroup>
+    <IntDir>$(Platform)/$(Configuration)/</IntDir>
+    <IncludePath>$(IncludePath)</IncludePath>
+    <CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRules />
+    <CodeAnalysisRuleAssemblies />
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Platform)'=='x64'">
+    <OutDir>../../../bin/win64/$(Configuration)/</OutDir>
+  </PropertyGroup>
+  <ItemDefinitionGroup>
+    <ClCompile>
+      <WarningLevel>Level3</WarningLevel>
+      <PreprocessorDefinitions>WIN32;_MBCS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <AdditionalIncludeDirectories>./;$(CudaToolkitDir)/include;../../../Common;</AdditionalIncludeDirectories>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <AdditionalDependencies>cudart_static.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalLibraryDirectories>$(CudaToolkitLibDir);</AdditionalLibraryDirectories>
+      <OutputFile>$(OutDir)/alignedTypes.exe</OutputFile>
+    </Link>
+    <CudaCompile>
+      <CodeGeneration>compute_35,sm_35;compute_37,sm_37;compute_50,sm_50;compute_52,sm_52;compute_60,sm_60;compute_61,sm_61;compute_70,sm_70;compute_75,sm_75;compute_80,sm_80;compute_86,sm_86;</CodeGeneration>
+      <AdditionalOptions>-Xcompiler "/wd 4819"  --threads 0 </AdditionalOptions>
+      <Include>./;../../../Common</Include>
+      <Defines>WIN32</Defines>
+    </CudaCompile>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)'=='Debug'">
+    <ClCompile>
+      <Optimization>Disabled</Optimization>
+      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <LinkTimeCodeGeneration>Default</LinkTimeCodeGeneration>
+    </Link>
+    <CudaCompile>
+      <Runtime>MTd</Runtime>
+      <TargetMachinePlatform>64</TargetMachinePlatform>
+    </CudaCompile>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)'=='Release'">
+    <ClCompile>
+      <Optimization>MaxSpeed</Optimization>
+      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <GenerateDebugInformation>false</GenerateDebugInformation>
+      <LinkTimeCodeGeneration>UseLinkTimeCodeGeneration</LinkTimeCodeGeneration>
+    </Link>
+    <CudaCompile>
+      <Runtime>MT</Runtime>
+      <TargetMachinePlatform>64</TargetMachinePlatform>
+    </CudaCompile>
+  </ItemDefinitionGroup>
+  <ItemGroup>
+    <CudaCompile Include="alignedTypes.cu" />
+
+  </ItemGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <ImportGroup Label="ExtensionTargets">
+    <Import Project="$(CUDAPropsPath)\CUDA 11.6.targets" />
+  </ImportGroup>
+</Project>

Samples/6_Performance/alignedTypes/alignedTypes_vs2022.sln

@@ -0,0 +1,20 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio 2022
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "alignedTypes", "alignedTypes_vs2022.vcxproj", "{997E0757-EA74-4A4E-A0FC-47D8C8831A15}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|x64 = Debug|x64
+		Release|x64 = Release|x64
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Debug|x64.ActiveCfg = Debug|x64
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Debug|x64.Build.0 = Debug|x64
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Release|x64.ActiveCfg = Release|x64
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Release|x64.Build.0 = Release|x64
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+EndGlobal

Samples/6_Performance/alignedTypes/alignedTypes_vs2022.vcxproj

@@ -0,0 +1,108 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <CUDAPropsPath Condition="'$(CUDAPropsPath)'==''">$(VCTargetsPath)\BuildCustomizations</CUDAPropsPath>
+  </PropertyGroup>
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Debug|x64">
+      <Configuration>Debug</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|x64">
+      <Configuration>Release</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>{997E0757-EA74-4A4E-A0FC-47D8C8831A15}</ProjectGuid>
+    <RootNamespace>alignedTypes_vs2022</RootNamespace>
+    <ProjectName>alignedTypes</ProjectName>
+    <CudaToolkitCustomDir />
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <PropertyGroup>
+    <ConfigurationType>Application</ConfigurationType>
+    <CharacterSet>MultiByte</CharacterSet>
+    <PlatformToolset>v143</PlatformToolset>
+    <WindowsTargetPlatformVersion>10.0</WindowsTargetPlatformVersion>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)'=='Debug'">
+    <UseDebugLibraries>true</UseDebugLibraries>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)'=='Release'">
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <ImportGroup Label="ExtensionSettings">
+    <Import Project="$(CUDAPropsPath)\CUDA 11.6.props" />
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets">
+    <Import Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" />
+  </ImportGroup>
+  <PropertyGroup Label="UserMacros" />
+  <PropertyGroup>
+    <IntDir>$(Platform)/$(Configuration)/</IntDir>
+    <IncludePath>$(IncludePath)</IncludePath>
+    <CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRules />
+    <CodeAnalysisRuleAssemblies />
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Platform)'=='x64'">
+    <OutDir>../../../bin/win64/$(Configuration)/</OutDir>
+  </PropertyGroup>
+  <ItemDefinitionGroup>
+    <ClCompile>
+      <WarningLevel>Level3</WarningLevel>
+      <PreprocessorDefinitions>WIN32;_MBCS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <AdditionalIncludeDirectories>./;$(CudaToolkitDir)/include;../../../Common;</AdditionalIncludeDirectories>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <AdditionalDependencies>cudart_static.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalLibraryDirectories>$(CudaToolkitLibDir);</AdditionalLibraryDirectories>
+      <OutputFile>$(OutDir)/alignedTypes.exe</OutputFile>
+    </Link>
+    <CudaCompile>
+      <CodeGeneration>compute_35,sm_35;compute_37,sm_37;compute_50,sm_50;compute_52,sm_52;compute_60,sm_60;compute_61,sm_61;compute_70,sm_70;compute_75,sm_75;compute_80,sm_80;compute_86,sm_86;</CodeGeneration>
+      <AdditionalOptions>-Xcompiler "/wd 4819"  --threads 0 </AdditionalOptions>
+      <Include>./;../../../Common</Include>
+      <Defines>WIN32</Defines>
+    </CudaCompile>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)'=='Debug'">
+    <ClCompile>
+      <Optimization>Disabled</Optimization>
+      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <LinkTimeCodeGeneration>Default</LinkTimeCodeGeneration>
+    </Link>
+    <CudaCompile>
+      <Runtime>MTd</Runtime>
+      <TargetMachinePlatform>64</TargetMachinePlatform>
+    </CudaCompile>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)'=='Release'">
+    <ClCompile>
+      <Optimization>MaxSpeed</Optimization>
+      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <GenerateDebugInformation>false</GenerateDebugInformation>
+      <LinkTimeCodeGeneration>UseLinkTimeCodeGeneration</LinkTimeCodeGeneration>
+    </Link>
+    <CudaCompile>
+      <Runtime>MT</Runtime>
+      <TargetMachinePlatform>64</TargetMachinePlatform>
+    </CudaCompile>
+  </ItemDefinitionGroup>
+  <ItemGroup>
+    <CudaCompile Include="alignedTypes.cu" />
+
+  </ItemGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <ImportGroup Label="ExtensionTargets">
+    <Import Project="$(CUDAPropsPath)\CUDA 11.6.targets" />
+  </ImportGroup>
+</Project>

Samples/6_Performance/alignedTypes/doc/alignedTypes.txt

@@ -0,0 +1,10 @@
+CUDA programming language, being a C with extensions, offers the ability to use arbitrary data structures in GPU programs. But in order for the hardware to perform efficient global loads and stores with variables of structured types, additional alignment details must be specified.
+
+Take a look at this structure definition:
+typedef struct{
+    float a;
+    float b;
+} testStructure;
+
+Without alignment specification the compiler will not automatically use a single 64-bit global memory load/store instruction, but will emit two 32-bit load instructions instead. 
+This significantly impacts aggregate load/store bandwidth, since the latter breaks coalescing rules because of incontiguous memory access pattern. Refer to section 5.1.2.1 of the Programming Guide.

Samples/6_Performance/transpose/.vscode/c_cpp_properties.json

@@ -0,0 +1,18 @@
+{
+    "configurations": [
+        {
+            "name": "Linux",
+            "includePath": [
+                "${workspaceFolder}/**",
+                "${workspaceFolder}/../../../Common"
+            ],
+            "defines": [],
+            "compilerPath": "/usr/local/cuda/bin/nvcc",
+            "cStandard": "gnu17",
+            "cppStandard": "gnu++14",
+            "intelliSenseMode": "linux-gcc-x64",
+            "configurationProvider": "ms-vscode.makefile-tools"
+        }
+    ],
+    "version": 4
+}

Samples/6_Performance/transpose/.vscode/extensions.json

@@ -0,0 +1,7 @@
+{
+    "recommendations": [
+        "nvidia.nsight-vscode-edition",
+        "ms-vscode.cpptools",
+        "ms-vscode.makefile-tools"
+    ]
+}

Samples/6_Performance/transpose/.vscode/launch.json

@@ -0,0 +1,10 @@
+{
+    "configurations": [
+        {
+            "name": "CUDA C++: Launch",
+            "type": "cuda-gdb",
+            "request": "launch",
+            "program": "${workspaceFolder}/transpose"
+        }
+    ]
+}

Samples/6_Performance/transpose/.vscode/tasks.json

@@ -0,0 +1,15 @@
+{
+    "version": "2.0.0",
+    "tasks": [
+        {
+            "label": "sample",
+            "type": "shell",
+            "command": "make dbg=1",
+            "problemMatcher": ["$nvcc"],
+            "group": {
+                "kind": "build",
+                "isDefault": true
+            }
+        }
+    ]
+}

Samples/6_Performance/transpose/Makefile

@@ -0,0 +1,340 @@
+################################################################################
+# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#  * Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#  * Neither the name of NVIDIA CORPORATION nor the names of its
+#    contributors may be used to endorse or promote products derived
+#    from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+################################################################################
+#
+# Makefile project only supported on Mac OS X and Linux Platforms)
+#
+################################################################################
+
+# Location of the CUDA Toolkit
+CUDA_PATH ?= /usr/local/cuda
+
+##############################
+# start deprecated interface #
+##############################
+ifeq ($(x86_64),1)
+    $(info WARNING - x86_64 variable has been deprecated)
+    $(info WARNING - please use TARGET_ARCH=x86_64 instead)
+    TARGET_ARCH ?= x86_64
+endif
+ifeq ($(ARMv7),1)
+    $(info WARNING - ARMv7 variable has been deprecated)
+    $(info WARNING - please use TARGET_ARCH=armv7l instead)
+    TARGET_ARCH ?= armv7l
+endif
+ifeq ($(aarch64),1)
+    $(info WARNING - aarch64 variable has been deprecated)
+    $(info WARNING - please use TARGET_ARCH=aarch64 instead)
+    TARGET_ARCH ?= aarch64
+endif
+ifeq ($(ppc64le),1)
+    $(info WARNING - ppc64le variable has been deprecated)
+    $(info WARNING - please use TARGET_ARCH=ppc64le instead)
+    TARGET_ARCH ?= ppc64le
+endif
+ifneq ($(GCC),)
+    $(info WARNING - GCC variable has been deprecated)
+    $(info WARNING - please use HOST_COMPILER=$(GCC) instead)
+    HOST_COMPILER ?= $(GCC)
+endif
+ifneq ($(abi),)
+    $(error ERROR - abi variable has been removed)
+endif
+############################
+# end deprecated interface #
+############################
+
+# architecture
+HOST_ARCH   := $(shell uname -m)
+TARGET_ARCH ?= $(HOST_ARCH)
+ifneq (,$(filter $(TARGET_ARCH),x86_64 aarch64 sbsa ppc64le armv7l))
+    ifneq ($(TARGET_ARCH),$(HOST_ARCH))
+        ifneq (,$(filter $(TARGET_ARCH),x86_64 aarch64 sbsa ppc64le))
+            TARGET_SIZE := 64
+        else ifneq (,$(filter $(TARGET_ARCH),armv7l))
+            TARGET_SIZE := 32
+        endif
+    else
+        TARGET_SIZE := $(shell getconf LONG_BIT)
+    endif
+else
+    $(error ERROR - unsupported value $(TARGET_ARCH) for TARGET_ARCH!)
+endif
+
+# sbsa and aarch64 systems look similar. Need to differentiate them at host level for now.
+ifeq ($(HOST_ARCH),aarch64)
+    ifeq ($(CUDA_PATH)/targets/sbsa-linux,$(shell ls -1d $(CUDA_PATH)/targets/sbsa-linux 2>/dev/null))
+        HOST_ARCH := sbsa
+        TARGET_ARCH := sbsa
+    endif
+endif
+
+ifneq ($(TARGET_ARCH),$(HOST_ARCH))
+    ifeq (,$(filter $(HOST_ARCH)-$(TARGET_ARCH),aarch64-armv7l x86_64-armv7l x86_64-aarch64 x86_64-sbsa x86_64-ppc64le))
+        $(error ERROR - cross compiling from $(HOST_ARCH) to $(TARGET_ARCH) is not supported!)
+    endif
+endif
+
+# When on native aarch64 system with userspace of 32-bit, change TARGET_ARCH to armv7l
+ifeq ($(HOST_ARCH)-$(TARGET_ARCH)-$(TARGET_SIZE),aarch64-aarch64-32)
+    TARGET_ARCH = armv7l
+endif
+
+# operating system
+HOST_OS   := $(shell uname -s 2>/dev/null | tr "[:upper:]" "[:lower:]")
+TARGET_OS ?= $(HOST_OS)
+ifeq (,$(filter $(TARGET_OS),linux darwin qnx android))
+    $(error ERROR - unsupported value $(TARGET_OS) for TARGET_OS!)
+endif
+
+# host compiler
+ifeq ($(TARGET_OS),darwin)
+    ifeq ($(shell expr `xcodebuild -version | grep -i xcode | awk '{print $$2}' | cut -d'.' -f1` \>= 5),1)
+        HOST_COMPILER ?= clang++
+    endif
+else ifneq ($(TARGET_ARCH),$(HOST_ARCH))
+    ifeq ($(HOST_ARCH)-$(TARGET_ARCH),x86_64-armv7l)
+        ifeq ($(TARGET_OS),linux)
+            HOST_COMPILER ?= arm-linux-gnueabihf-g++
+        else ifeq ($(TARGET_OS),qnx)
+            ifeq ($(QNX_HOST),)
+                $(error ERROR - QNX_HOST must be passed to the QNX host toolchain)
+            endif
+            ifeq ($(QNX_TARGET),)
+                $(error ERROR - QNX_TARGET must be passed to the QNX target toolchain)
+            endif
+            export QNX_HOST
+            export QNX_TARGET
+            HOST_COMPILER ?= $(QNX_HOST)/usr/bin/arm-unknown-nto-qnx6.6.0eabi-g++
+        else ifeq ($(TARGET_OS),android)
+            HOST_COMPILER ?= arm-linux-androideabi-g++
+        endif
+    else ifeq ($(TARGET_ARCH),aarch64)
+        ifeq ($(TARGET_OS), linux)
+            HOST_COMPILER ?= aarch64-linux-gnu-g++
+        else ifeq ($(TARGET_OS),qnx)
+            ifeq ($(QNX_HOST),)
+                $(error ERROR - QNX_HOST must be passed to the QNX host toolchain)
+            endif
+            ifeq ($(QNX_TARGET),)
+                $(error ERROR - QNX_TARGET must be passed to the QNX target toolchain)
+            endif
+            export QNX_HOST
+            export QNX_TARGET
+            HOST_COMPILER ?= $(QNX_HOST)/usr/bin/q++
+        else ifeq ($(TARGET_OS), android)
+            HOST_COMPILER ?= aarch64-linux-android-clang++
+        endif
+    else ifeq ($(TARGET_ARCH),sbsa)
+        HOST_COMPILER ?= aarch64-linux-gnu-g++
+    else ifeq ($(TARGET_ARCH),ppc64le)
+        HOST_COMPILER ?= powerpc64le-linux-gnu-g++
+    endif
+endif
+HOST_COMPILER ?= g++
+NVCC          := $(CUDA_PATH)/bin/nvcc -ccbin $(HOST_COMPILER)
+
+# internal flags
+NVCCFLAGS   := -m${TARGET_SIZE}
+CCFLAGS     :=
+LDFLAGS     :=
+
+# build flags
+ifeq ($(TARGET_OS),darwin)
+    LDFLAGS += -rpath $(CUDA_PATH)/lib
+    CCFLAGS += -arch $(HOST_ARCH)
+else ifeq ($(HOST_ARCH)-$(TARGET_ARCH)-$(TARGET_OS),x86_64-armv7l-linux)
+    LDFLAGS += --dynamic-linker=/lib/ld-linux-armhf.so.3
+    CCFLAGS += -mfloat-abi=hard
+else ifeq ($(TARGET_OS),android)
+    LDFLAGS += -pie
+    CCFLAGS += -fpie -fpic -fexceptions
+endif
+
+ifneq ($(TARGET_ARCH),$(HOST_ARCH))
+    ifeq ($(TARGET_ARCH)-$(TARGET_OS),armv7l-linux)
+        ifneq ($(TARGET_FS),)
+            GCCVERSIONLTEQ46 := $(shell expr `$(HOST_COMPILER) -dumpversion` \<= 4.6)
+            ifeq ($(GCCVERSIONLTEQ46),1)
+                CCFLAGS += --sysroot=$(TARGET_FS)
+            endif
+            LDFLAGS += --sysroot=$(TARGET_FS)
+            LDFLAGS += -rpath-link=$(TARGET_FS)/lib
+            LDFLAGS += -rpath-link=$(TARGET_FS)/usr/lib
+            LDFLAGS += -rpath-link=$(TARGET_FS)/usr/lib/arm-linux-gnueabihf
+        endif
+    endif
+    ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-linux)
+        ifneq ($(TARGET_FS),)
+            GCCVERSIONLTEQ46 := $(shell expr `$(HOST_COMPILER) -dumpversion` \<= 4.6)
+            ifeq ($(GCCVERSIONLTEQ46),1)
+                CCFLAGS += --sysroot=$(TARGET_FS)
+            endif
+            LDFLAGS += --sysroot=$(TARGET_FS)
+            LDFLAGS += -rpath-link=$(TARGET_FS)/lib -L$(TARGET_FS)/lib
+            LDFLAGS += -rpath-link=$(TARGET_FS)/lib/aarch64-linux-gnu -L$(TARGET_FS)/lib/aarch64-linux-gnu
+            LDFLAGS += -rpath-link=$(TARGET_FS)/usr/lib -L$(TARGET_FS)/usr/lib
+            LDFLAGS += -rpath-link=$(TARGET_FS)/usr/lib/aarch64-linux-gnu -L$(TARGET_FS)/usr/lib/aarch64-linux-gnu
+            LDFLAGS += --unresolved-symbols=ignore-in-shared-libs
+            CCFLAGS += -isystem=$(TARGET_FS)/usr/include -I$(TARGET_FS)/usr/include -I$(TARGET_FS)/usr/include/libdrm
+            CCFLAGS += -isystem=$(TARGET_FS)/usr/include/aarch64-linux-gnu -I$(TARGET_FS)/usr/include/aarch64-linux-gnu
+        endif
+    endif
+    ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-qnx)
+        NVCCFLAGS += -D_QNX_SOURCE
+        NVCCFLAGS += --qpp-config 8.3.0,gcc_ntoaarch64le
+        CCFLAGS += -DWIN_INTERFACE_CUSTOM -I/usr/include/aarch64-qnx-gnu
+        LDFLAGS += -lsocket
+        LDFLAGS += -L/usr/lib/aarch64-qnx-gnu
+        CCFLAGS += "-Wl\,-rpath-link\,/usr/lib/aarch64-qnx-gnu"
+        ifdef TARGET_OVERRIDE
+            LDFLAGS += -lslog2
+        endif
+
+        ifneq ($(TARGET_FS),)
+            LDFLAGS += -L$(TARGET_FS)/usr/lib
+            CCFLAGS += "-Wl\,-rpath-link\,$(TARGET_FS)/usr/lib"
+            LDFLAGS += -L$(TARGET_FS)/usr/libnvidia
+            CCFLAGS += "-Wl\,-rpath-link\,$(TARGET_FS)/usr/libnvidia"
+            CCFLAGS += -I$(TARGET_FS)/../include
+        endif
+    endif
+endif
+
+ifdef TARGET_OVERRIDE # cuda toolkit targets override
+    NVCCFLAGS += -target-dir $(TARGET_OVERRIDE)
+endif
+
+# Install directory of different arch
+CUDA_INSTALL_TARGET_DIR :=
+ifeq ($(TARGET_ARCH)-$(TARGET_OS),armv7l-linux)
+    CUDA_INSTALL_TARGET_DIR = targets/armv7-linux-gnueabihf/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-linux)
+    CUDA_INSTALL_TARGET_DIR = targets/aarch64-linux/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),sbsa-linux)
+    CUDA_INSTALL_TARGET_DIR = targets/sbsa-linux/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),armv7l-android)
+    CUDA_INSTALL_TARGET_DIR = targets/armv7-linux-androideabi/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-android)
+    CUDA_INSTALL_TARGET_DIR = targets/aarch64-linux-androideabi/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),armv7l-qnx)
+    CUDA_INSTALL_TARGET_DIR = targets/ARMv7-linux-QNX/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-qnx)
+    CUDA_INSTALL_TARGET_DIR = targets/aarch64-qnx/
+else ifeq ($(TARGET_ARCH),ppc64le)
+    CUDA_INSTALL_TARGET_DIR = targets/ppc64le-linux/
+endif
+
+# Debug build flags
+ifeq ($(dbg),1)
+      NVCCFLAGS += -g -G
+      BUILD_TYPE := debug
+else
+      BUILD_TYPE := release
+endif
+
+ALL_CCFLAGS :=
+ALL_CCFLAGS += $(NVCCFLAGS)
+ALL_CCFLAGS += $(EXTRA_NVCCFLAGS)
+ALL_CCFLAGS += $(addprefix -Xcompiler ,$(CCFLAGS))
+ALL_CCFLAGS += $(addprefix -Xcompiler ,$(EXTRA_CCFLAGS))
+
+SAMPLE_ENABLED := 1
+
+ALL_LDFLAGS :=
+ALL_LDFLAGS += $(ALL_CCFLAGS)
+ALL_LDFLAGS += $(addprefix -Xlinker ,$(LDFLAGS))
+ALL_LDFLAGS += $(addprefix -Xlinker ,$(EXTRA_LDFLAGS))
+
+# Common includes and paths for CUDA
+INCLUDES  := -I../../../Common
+LIBRARIES :=
+
+################################################################################
+
+# Gencode arguments
+ifeq ($(TARGET_ARCH),$(filter $(TARGET_ARCH),armv7l aarch64 sbsa))
+SMS ?= 53 61 70 72 75 80 86 87
+else
+SMS ?= 35 37 50 52 60 61 70 75 80 86
+endif
+
+ifeq ($(SMS),)
+$(info >>> WARNING - no SM architectures have been specified - waiving sample <<<)
+SAMPLE_ENABLED := 0
+endif
+
+ifeq ($(GENCODE_FLAGS),)
+# Generate SASS code for each SM architecture listed in $(SMS)
+$(foreach sm,$(SMS),$(eval GENCODE_FLAGS += -gencode arch=compute_$(sm),code=sm_$(sm)))
+
+# Generate PTX code from the highest SM architecture in $(SMS) to guarantee forward-compatibility
+HIGHEST_SM := $(lastword $(sort $(SMS)))
+ifneq ($(HIGHEST_SM),)
+GENCODE_FLAGS += -gencode arch=compute_$(HIGHEST_SM),code=compute_$(HIGHEST_SM)
+endif
+endif
+
+ALL_CCFLAGS += --threads 0 --std=c++11
+
+ifeq ($(SAMPLE_ENABLED),0)
+EXEC ?= @echo "[@]"
+endif
+
+################################################################################
+
+# Target rules
+all: build
+
+build: transpose
+
+check.deps:
+ifeq ($(SAMPLE_ENABLED),0)
+	@echo "Sample will be waived due to the above missing dependencies"
+else
+	@echo "Sample is ready - all dependencies have been met"
+endif
+
+transpose.o:transpose.cu
+	$(EXEC) $(NVCC) $(INCLUDES) $(ALL_CCFLAGS) $(GENCODE_FLAGS) -o $@ -c $<
+
+transpose: transpose.o
+	$(EXEC) $(NVCC) $(ALL_LDFLAGS) $(GENCODE_FLAGS) -o $@ $+ $(LIBRARIES)
+	$(EXEC) mkdir -p ../../../bin/$(TARGET_ARCH)/$(TARGET_OS)/$(BUILD_TYPE)
+	$(EXEC) cp $@ ../../../bin/$(TARGET_ARCH)/$(TARGET_OS)/$(BUILD_TYPE)
+
+run: build
+	$(EXEC) ./transpose
+
+testrun: build
+
+clean:
+	rm -f transpose transpose.o
+	rm -rf ../../../bin/$(TARGET_ARCH)/$(TARGET_OS)/$(BUILD_TYPE)/transpose
+
+clobber: clean

Samples/6_Performance/transpose/NsightEclipse.xml

@@ -0,0 +1,87 @@
+<?xml version="1.0" encoding="UTF-8"?> 
+<!DOCTYPE entry SYSTEM "SamplesInfo.dtd">
+<entry>
+  <name>transpose</name>
+  <cuda_api_list>
+    <toolkit>cudaFree</toolkit>
+    <toolkit>cudaEventRecord</toolkit>
+    <toolkit>cudaEventCreate</toolkit>
+    <toolkit>cudaEventElapsedTime</toolkit>
+    <toolkit>cudaEventSynchronize</toolkit>
+    <toolkit>cudaMalloc</toolkit>
+    <toolkit>cudaEventDestroy</toolkit>
+    <toolkit>cudaGetLastError</toolkit>
+    <toolkit>cudaMemcpy</toolkit>
+    <toolkit>cudaGetDeviceProperties</toolkit>
+    <toolkit>cudaGetDevice</toolkit>
+  </cuda_api_list>
+  <description><![CDATA[This sample demonstrates Matrix Transpose.  Different performance are shown to achieve high performance.]]></description>
+  <devicecompilation>whole</devicecompilation>
+  <includepaths>
+    <path>./</path>
+    <path>../</path>
+    <path>../../../Common</path>
+  </includepaths>
+  <keyconcepts>
+    <concept level="advanced">Performance Strategies</concept>
+    <concept level="advanced">Linear Algebra</concept>
+  </keyconcepts>
+  <keywords>
+    <keyword>GPGPU</keyword>
+    <keyword>matrix</keyword>
+    <keyword>transpose</keyword>
+  </keywords>
+  <libraries>
+  </libraries>
+  <librarypaths>
+  </librarypaths>
+  <nsight_eclipse>true</nsight_eclipse>
+  <primary_file>transpose.cu</primary_file>
+  <scopes>
+    <scope>1:CUDA Advanced Topics</scope>
+    <scope>1:Performance Strategies</scope>
+    <scope>3:Linear Algebra</scope>
+  </scopes>
+  <sm-arch>sm35</sm-arch>
+  <sm-arch>sm37</sm-arch>
+  <sm-arch>sm50</sm-arch>
+  <sm-arch>sm52</sm-arch>
+  <sm-arch>sm53</sm-arch>
+  <sm-arch>sm60</sm-arch>
+  <sm-arch>sm61</sm-arch>
+  <sm-arch>sm70</sm-arch>
+  <sm-arch>sm72</sm-arch>
+  <sm-arch>sm75</sm-arch>
+  <sm-arch>sm80</sm-arch>
+  <sm-arch>sm86</sm-arch>
+  <sm-arch>sm87</sm-arch>
+  <supported_envs>
+    <env>
+      <arch>x86_64</arch>
+      <platform>linux</platform>
+    </env>
+    <env>
+      <platform>windows7</platform>
+    </env>
+    <env>
+      <arch>x86_64</arch>
+      <platform>macosx</platform>
+    </env>
+    <env>
+      <arch>arm</arch>
+    </env>
+    <env>
+      <arch>sbsa</arch>
+    </env>
+    <env>
+      <arch>ppc64le</arch>
+      <platform>linux</platform>
+    </env>
+  </supported_envs>
+  <supported_sm_architectures>
+    <include>all</include>
+  </supported_sm_architectures>
+  <title>Matrix Transpose</title>
+  <type>exe</type>
+  <whitepaper>doc\MatrixTranspose.pdf</whitepaper>
+</entry>

Samples/6_Performance/transpose/README.md

@@ -0,0 +1,71 @@
+# transpose - Matrix Transpose
+
+## Description
+
+This sample demonstrates Matrix Transpose.  Different performance are shown to achieve high performance.
+
+## Key Concepts
+
+Performance Strategies, Linear Algebra
+
+## Supported SM Architectures
+
+[SM 3.5 ](https://developer.nvidia.com/cuda-gpus)  [SM 3.7 ](https://developer.nvidia.com/cuda-gpus)  [SM 5.0 ](https://developer.nvidia.com/cuda-gpus)  [SM 5.2 ](https://developer.nvidia.com/cuda-gpus)  [SM 5.3 ](https://developer.nvidia.com/cuda-gpus)  [SM 6.0 ](https://developer.nvidia.com/cuda-gpus)  [SM 6.1 ](https://developer.nvidia.com/cuda-gpus)  [SM 7.0 ](https://developer.nvidia.com/cuda-gpus)  [SM 7.2 ](https://developer.nvidia.com/cuda-gpus)  [SM 7.5 ](https://developer.nvidia.com/cuda-gpus)  [SM 8.0 ](https://developer.nvidia.com/cuda-gpus)  [SM 8.6 ](https://developer.nvidia.com/cuda-gpus)  [SM 8.7 ](https://developer.nvidia.com/cuda-gpus)
+
+## Supported OSes
+
+Linux, Windows
+
+## Supported CPU Architecture
+
+x86_64, ppc64le, armv7l
+
+## CUDA APIs involved
+
+### [CUDA Runtime API](http://docs.nvidia.com/cuda/cuda-runtime-api/index.html)
+cudaFree, cudaEventRecord, cudaEventCreate, cudaEventElapsedTime, cudaEventSynchronize, cudaMalloc, cudaEventDestroy, cudaGetLastError, cudaMemcpy, cudaGetDeviceProperties, cudaGetDevice
+
+## Prerequisites
+
+Download and install the [CUDA Toolkit 11.6](https://developer.nvidia.com/cuda-downloads) for your corresponding platform.
+
+## Build and Run
+
+### Windows
+The Windows samples are built using the Visual Studio IDE. Solution files (.sln) are provided for each supported version of Visual Studio, using the format:
+```
+*_vs<version>.sln - for Visual Studio <version>
+```
+Each individual sample has its own set of solution files in its directory:
+
+To build/examine all the samples at once, the complete solution files should be used. To build/examine a single sample, the individual sample solution files should be used.
+> **Note:** Some samples require that the Microsoft DirectX SDK (June 2010 or newer) be installed and that the VC++ directory paths are properly set up (**Tools > Options...**). Check DirectX Dependencies section for details."
+
+### Linux
+The Linux samples are built using makefiles. To use the makefiles, change the current directory to the sample directory you wish to build, and run make:
+```
+$ cd <sample_dir>
+$ make
+```
+The samples makefiles can take advantage of certain options:
+*  **TARGET_ARCH=<arch>** - cross-compile targeting a specific architecture. Allowed architectures are x86_64, ppc64le, armv7l.
+    By default, TARGET_ARCH is set to HOST_ARCH. On a x86_64 machine, not setting TARGET_ARCH is the equivalent of setting TARGET_ARCH=x86_64.<br/>
+`$ make TARGET_ARCH=x86_64` <br/> `$ make TARGET_ARCH=ppc64le` <br/> `$ make TARGET_ARCH=armv7l` <br/>
+    See [here](http://docs.nvidia.com/cuda/cuda-samples/index.html#cross-samples) for more details.
+*   **dbg=1** - build with debug symbols
+    ```
+    $ make dbg=1
+    ```
+*   **SMS="A B ..."** - override the SM architectures for which the sample will be built, where `"A B ..."` is a space-delimited list of SM architectures. For example, to generate SASS for SM 50 and SM 60, use `SMS="50 60"`.
+    ```
+    $ make SMS="50 60"
+    ```
+
+*  **HOST_COMPILER=<host_compiler>** - override the default g++ host compiler. See the [Linux Installation Guide](http://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html#system-requirements) for a list of supported host compilers.
+```
+    $ make HOST_COMPILER=g++
+```
+
+## References (for more details)
+
+[whitepaper](./doc/MatrixTranspose.pdf)

Samples/6_Performance/transpose/transpose.cu

@@ -0,0 +1,613 @@
+/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+// -----------------------------------------------------------------------------
+// Transpose
+//
+// This file contains both device and host code for transposing a floating-point
+// matrix.  It performs several transpose kernels, which incrementally improve
+// performance through coalescing, removing shared memory bank conflicts, and
+// eliminating partition camping.  Several of the kernels perform a copy, used
+// to represent the best case performance that a transpose can achieve.
+//
+// Please see the whitepaper in the docs folder of the transpose project for a
+// detailed description of this performance study.
+// -----------------------------------------------------------------------------
+
+#include <cooperative_groups.h>
+
+namespace cg = cooperative_groups;
+// Utilities and system includes
+#include <helper_string.h>    // helper for string parsing
+#include <helper_image.h>     // helper for image and data comparison
+#include <helper_cuda.h>      // helper for cuda error checking functions
+
+const char *sSDKsample = "Transpose";
+
+// Each block transposes/copies a tile of TILE_DIM x TILE_DIM elements
+// using TILE_DIM x BLOCK_ROWS threads, so that each thread transposes
+// TILE_DIM/BLOCK_ROWS elements.  TILE_DIM must be an integral multiple of
+// BLOCK_ROWS
+
+#define TILE_DIM 16
+#define BLOCK_ROWS 16
+
+// This sample assumes that MATRIX_SIZE_X = MATRIX_SIZE_Y
+int MATRIX_SIZE_X = 1024;
+int MATRIX_SIZE_Y = 1024;
+int MUL_FACTOR = TILE_DIM;
+
+#define FLOOR(a, b) (a - (a % b))
+
+// Compute the tile size necessary to illustrate performance cases for SM20+
+// hardware
+int MAX_TILES = (FLOOR(MATRIX_SIZE_X, 512) * FLOOR(MATRIX_SIZE_Y, 512)) /
+                (TILE_DIM * TILE_DIM);
+
+// Number of repetitions used for timing.  Two sets of repetitions are
+// performed: 1) over kernel launches and 2) inside the kernel over just the
+// loads and stores
+
+#define NUM_REPS 100
+
+// -------------------------------------------------------
+// Copies
+// width and height must be integral multiples of TILE_DIM
+// -------------------------------------------------------
+
+__global__ void copy(float *odata, float *idata, int width, int height) {
+  int xIndex = blockIdx.x * TILE_DIM + threadIdx.x;
+  int yIndex = blockIdx.y * TILE_DIM + threadIdx.y;
+
+  int index = xIndex + width * yIndex;
+
+  for (int i = 0; i < TILE_DIM; i += BLOCK_ROWS) {
+    odata[index + i * width] = idata[index + i * width];
+  }
+}
+
+__global__ void copySharedMem(float *odata, float *idata, int width,
+                              int height) {
+  // Handle to thread block group
+  cg::thread_block cta = cg::this_thread_block();
+  __shared__ float tile[TILE_DIM][TILE_DIM];
+
+  int xIndex = blockIdx.x * TILE_DIM + threadIdx.x;
+  int yIndex = blockIdx.y * TILE_DIM + threadIdx.y;
+
+  int index = xIndex + width * yIndex;
+
+  for (int i = 0; i < TILE_DIM; i += BLOCK_ROWS) {
+    if (xIndex < width && yIndex < height) {
+      tile[threadIdx.y][threadIdx.x] = idata[index];
+    }
+  }
+
+  cg::sync(cta);
+
+  for (int i = 0; i < TILE_DIM; i += BLOCK_ROWS) {
+    if (xIndex < height && yIndex < width) {
+      odata[index] = tile[threadIdx.y][threadIdx.x];
+    }
+  }
+}
+
+// -------------------------------------------------------
+// Transposes
+// width and height must be integral multiples of TILE_DIM
+// -------------------------------------------------------
+
+__global__ void transposeNaive(float *odata, float *idata, int width,
+                               int height) {
+  int xIndex = blockIdx.x * TILE_DIM + threadIdx.x;
+  int yIndex = blockIdx.y * TILE_DIM + threadIdx.y;
+
+  int index_in = xIndex + width * yIndex;
+  int index_out = yIndex + height * xIndex;
+
+  for (int i = 0; i < TILE_DIM; i += BLOCK_ROWS) {
+    odata[index_out + i] = idata[index_in + i * width];
+  }
+}
+
+// coalesced transpose (with bank conflicts)
+
+__global__ void transposeCoalesced(float *odata, float *idata, int width,
+                                   int height) {
+  // Handle to thread block group
+  cg::thread_block cta = cg::this_thread_block();
+  __shared__ float tile[TILE_DIM][TILE_DIM];
+
+  int xIndex = blockIdx.x * TILE_DIM + threadIdx.x;
+  int yIndex = blockIdx.y * TILE_DIM + threadIdx.y;
+  int index_in = xIndex + (yIndex)*width;
+
+  xIndex = blockIdx.y * TILE_DIM + threadIdx.x;
+  yIndex = blockIdx.x * TILE_DIM + threadIdx.y;
+  int index_out = xIndex + (yIndex)*height;
+
+  for (int i = 0; i < TILE_DIM; i += BLOCK_ROWS) {
+    tile[threadIdx.y + i][threadIdx.x] = idata[index_in + i * width];
+  }
+
+  cg::sync(cta);
+
+  for (int i = 0; i < TILE_DIM; i += BLOCK_ROWS) {
+    odata[index_out + i * height] = tile[threadIdx.x][threadIdx.y + i];
+  }
+}
+
+// Coalesced transpose with no bank conflicts
+
+__global__ void transposeNoBankConflicts(float *odata, float *idata, int width,
+                                         int height) {
+  // Handle to thread block group
+  cg::thread_block cta = cg::this_thread_block();
+  __shared__ float tile[TILE_DIM][TILE_DIM + 1];
+
+  int xIndex = blockIdx.x * TILE_DIM + threadIdx.x;
+  int yIndex = blockIdx.y * TILE_DIM + threadIdx.y;
+  int index_in = xIndex + (yIndex)*width;
+
+  xIndex = blockIdx.y * TILE_DIM + threadIdx.x;
+  yIndex = blockIdx.x * TILE_DIM + threadIdx.y;
+  int index_out = xIndex + (yIndex)*height;
+
+  for (int i = 0; i < TILE_DIM; i += BLOCK_ROWS) {
+    tile[threadIdx.y + i][threadIdx.x] = idata[index_in + i * width];
+  }
+
+  cg::sync(cta);
+
+  for (int i = 0; i < TILE_DIM; i += BLOCK_ROWS) {
+    odata[index_out + i * height] = tile[threadIdx.x][threadIdx.y + i];
+  }
+}
+
+// Transpose that effectively reorders execution of thread blocks along
+// diagonals of the matrix (also coalesced and has no bank conflicts)
+//
+// Here blockIdx.x is interpreted as the distance along a diagonal and
+// blockIdx.y as corresponding to different diagonals
+//
+// blockIdx_x and blockIdx_y expressions map the diagonal coordinates to the
+// more commonly used cartesian coordinates so that the only changes to the code
+// from the coalesced version are the calculation of the blockIdx_x and
+// blockIdx_y and replacement of blockIdx.x and bloclIdx.y with the subscripted
+// versions in the remaining code
+
+__global__ void transposeDiagonal(float *odata, float *idata, int width,
+                                  int height) {
+  // Handle to thread block group
+  cg::thread_block cta = cg::this_thread_block();
+  __shared__ float tile[TILE_DIM][TILE_DIM + 1];
+
+  int blockIdx_x, blockIdx_y;
+
+  // do diagonal reordering
+  if (width == height) {
+    blockIdx_y = blockIdx.x;
+    blockIdx_x = (blockIdx.x + blockIdx.y) % gridDim.x;
+  } else {
+    int bid = blockIdx.x + gridDim.x * blockIdx.y;
+    blockIdx_y = bid % gridDim.y;
+    blockIdx_x = ((bid / gridDim.y) + blockIdx_y) % gridDim.x;
+  }
+
+  // from here on the code is same as previous kernel except blockIdx_x replaces
+  // blockIdx.x and similarly for y
+
+  int xIndex = blockIdx_x * TILE_DIM + threadIdx.x;
+  int yIndex = blockIdx_y * TILE_DIM + threadIdx.y;
+  int index_in = xIndex + (yIndex)*width;
+
+  xIndex = blockIdx_y * TILE_DIM + threadIdx.x;
+  yIndex = blockIdx_x * TILE_DIM + threadIdx.y;
+  int index_out = xIndex + (yIndex)*height;
+
+  for (int i = 0; i < TILE_DIM; i += BLOCK_ROWS) {
+    tile[threadIdx.y + i][threadIdx.x] = idata[index_in + i * width];
+  }
+
+  cg::sync(cta);
+
+  for (int i = 0; i < TILE_DIM; i += BLOCK_ROWS) {
+    odata[index_out + i * height] = tile[threadIdx.x][threadIdx.y + i];
+  }
+}
+
+// --------------------------------------------------------------------
+// Partial transposes
+// NB: the coarse- and fine-grained routines only perform part of a
+//     transpose and will fail the test against the reference solution
+//
+//     They are used to assess performance characteristics of different
+//     components of a full transpose
+// --------------------------------------------------------------------
+
+__global__ void transposeFineGrained(float *odata, float *idata, int width,
+                                     int height) {
+  // Handle to thread block group
+  cg::thread_block cta = cg::this_thread_block();
+  __shared__ float block[TILE_DIM][TILE_DIM + 1];
+
+  int xIndex = blockIdx.x * TILE_DIM + threadIdx.x;
+  int yIndex = blockIdx.y * TILE_DIM + threadIdx.y;
+  int index = xIndex + (yIndex)*width;
+
+  for (int i = 0; i < TILE_DIM; i += BLOCK_ROWS) {
+    block[threadIdx.y + i][threadIdx.x] = idata[index + i * width];
+  }
+
+  cg::sync(cta);
+
+  for (int i = 0; i < TILE_DIM; i += BLOCK_ROWS) {
+    odata[index + i * height] = block[threadIdx.x][threadIdx.y + i];
+  }
+}
+
+__global__ void transposeCoarseGrained(float *odata, float *idata, int width,
+                                       int height) {
+  // Handle to thread block group
+  cg::thread_block cta = cg::this_thread_block();
+  __shared__ float block[TILE_DIM][TILE_DIM + 1];
+
+  int xIndex = blockIdx.x * TILE_DIM + threadIdx.x;
+  int yIndex = blockIdx.y * TILE_DIM + threadIdx.y;
+  int index_in = xIndex + (yIndex)*width;
+
+  xIndex = blockIdx.y * TILE_DIM + threadIdx.x;
+  yIndex = blockIdx.x * TILE_DIM + threadIdx.y;
+  int index_out = xIndex + (yIndex)*height;
+
+  for (int i = 0; i < TILE_DIM; i += BLOCK_ROWS) {
+    block[threadIdx.y + i][threadIdx.x] = idata[index_in + i * width];
+  }
+
+  cg::sync(cta);
+
+  for (int i = 0; i < TILE_DIM; i += BLOCK_ROWS) {
+    odata[index_out + i * height] = block[threadIdx.y + i][threadIdx.x];
+  }
+}
+
+// ---------------------
+// host utility routines
+// ---------------------
+
+void computeTransposeGold(float *gold, float *idata, const int size_x,
+                          const int size_y) {
+  for (int y = 0; y < size_y; ++y) {
+    for (int x = 0; x < size_x; ++x) {
+      gold[(x * size_y) + y] = idata[(y * size_x) + x];
+    }
+  }
+}
+
+void getParams(int argc, char **argv, cudaDeviceProp &deviceProp, int &size_x,
+               int &size_y, int max_tile_dim) {
+  // set matrix size (if (x,y) dim of matrix is not square, then this will have
+  // to be modified
+  if (checkCmdLineFlag(argc, (const char **)argv, "dimX")) {
+    size_x = getCmdLineArgumentInt(argc, (const char **)argv, "dimX");
+
+    if (size_x > max_tile_dim) {
+      printf("> MatrixSize X = %d is greater than the recommended size = %d\n",
+             size_x, max_tile_dim);
+    } else {
+      printf("> MatrixSize X = %d\n", size_x);
+    }
+  } else {
+    size_x = max_tile_dim;
+    size_x = FLOOR(size_x, 512);
+  }
+
+  if (checkCmdLineFlag(argc, (const char **)argv, "dimY")) {
+    size_y = getCmdLineArgumentInt(argc, (const char **)argv, "dimY");
+
+    if (size_y > max_tile_dim) {
+      printf("> MatrixSize Y = %d is greater than the recommended size = %d\n",
+             size_y, max_tile_dim);
+    } else {
+      printf("> MatrixSize Y = %d\n", size_y);
+    }
+  } else {
+    size_y = max_tile_dim;
+    size_y = FLOOR(size_y, 512);
+  }
+}
+
+void showHelp() {
+  printf("\n%s : Command line options\n", sSDKsample);
+  printf("\t-device=n          (where n=0,1,2.... for the GPU device)\n\n");
+  printf("> The default matrix size can be overridden with these parameters\n");
+  printf("\t-dimX=row_dim_size (matrix row    dimensions)\n");
+  printf("\t-dimY=col_dim_size (matrix column dimensions)\n");
+}
+
+// ----
+// main
+// ----
+
+int main(int argc, char **argv) {
+  // Start logs
+  printf("%s Starting...\n\n", sSDKsample);
+
+  if (checkCmdLineFlag(argc, (const char **)argv, "help")) {
+    showHelp();
+    return 0;
+  }
+
+  int devID = findCudaDevice(argc, (const char **)argv);
+  cudaDeviceProp deviceProp;
+
+  // get number of SMs on this GPU
+  checkCudaErrors(cudaGetDevice(&devID));
+  checkCudaErrors(cudaGetDeviceProperties(&deviceProp, devID));
+
+  // compute the scaling factor (for GPUs with fewer MPs)
+  float scale_factor, total_tiles;
+  scale_factor =
+      max((192.0f / (_ConvertSMVer2Cores(deviceProp.major, deviceProp.minor) *
+                     (float)deviceProp.multiProcessorCount)),
+          1.0f);
+
+  printf("> Device %d: \"%s\"\n", devID, deviceProp.name);
+  printf("> SM Capability %d.%d detected:\n", deviceProp.major,
+         deviceProp.minor);
+
+  // Calculate number of tiles we will run for the Matrix Transpose performance
+  // tests
+  int size_x, size_y, max_matrix_dim, matrix_size_test;
+
+  matrix_size_test = 512;  // we round down max_matrix_dim for this perf test
+  total_tiles = (float)MAX_TILES / scale_factor;
+
+  max_matrix_dim =
+      FLOOR((int)(floor(sqrt(total_tiles)) * TILE_DIM), matrix_size_test);
+
+  // This is the minimum size allowed
+  if (max_matrix_dim == 0) {
+    max_matrix_dim = matrix_size_test;
+  }
+
+  printf("> [%s] has %d MP(s) x %d (Cores/MP) = %d (Cores)\n", deviceProp.name,
+         deviceProp.multiProcessorCount,
+         _ConvertSMVer2Cores(deviceProp.major, deviceProp.minor),
+         _ConvertSMVer2Cores(deviceProp.major, deviceProp.minor) *
+             deviceProp.multiProcessorCount);
+
+  printf("> Compute performance scaling factor = %4.2f\n", scale_factor);
+
+  // Extract parameters if there are any, command line -dimx and -dimy can
+  // override any of these settings
+  getParams(argc, argv, deviceProp, size_x, size_y, max_matrix_dim);
+
+  if (size_x != size_y) {
+    printf(
+        "\n[%s] does not support non-square matrices (row_dim_size(%d) != "
+        "col_dim_size(%d))\nExiting...\n\n",
+        sSDKsample, size_x, size_y);
+    exit(EXIT_FAILURE);
+  }
+
+  if (size_x % TILE_DIM != 0 || size_y % TILE_DIM != 0) {
+    printf(
+        "[%s] Matrix size must be integral multiple of tile "
+        "size\nExiting...\n\n",
+        sSDKsample);
+    exit(EXIT_FAILURE);
+  }
+
+  // kernel pointer and descriptor
+  void (*kernel)(float *, float *, int, int);
+  const char *kernelName;
+
+  // execution configuration parameters
+  dim3 grid(size_x / TILE_DIM, size_y / TILE_DIM),
+      threads(TILE_DIM, BLOCK_ROWS);
+
+  if (grid.x < 1 || grid.y < 1) {
+    printf("[%s] grid size computation incorrect in test \nExiting...\n\n",
+           sSDKsample);
+    exit(EXIT_FAILURE);
+  }
+
+  // CUDA events
+  cudaEvent_t start, stop;
+
+  // size of memory required to store the matrix
+  size_t mem_size = static_cast<size_t>(sizeof(float) * size_x * size_y);
+
+  if (2 * mem_size > deviceProp.totalGlobalMem) {
+    printf("Input matrix size is larger than the available device memory!\n");
+    printf("Please choose a smaller size matrix\n");
+    exit(EXIT_FAILURE);
+  }
+
+  // allocate host memory
+  float *h_idata = (float *)malloc(mem_size);
+  float *h_odata = (float *)malloc(mem_size);
+  float *transposeGold = (float *)malloc(mem_size);
+  float *gold;
+
+  // allocate device memory
+  float *d_idata, *d_odata;
+  checkCudaErrors(cudaMalloc((void **)&d_idata, mem_size));
+  checkCudaErrors(cudaMalloc((void **)&d_odata, mem_size));
+
+  // initialize host data
+  for (int i = 0; i < (size_x * size_y); ++i) {
+    h_idata[i] = (float)i;
+  }
+
+  // copy host data to device
+  checkCudaErrors(
+      cudaMemcpy(d_idata, h_idata, mem_size, cudaMemcpyHostToDevice));
+
+  // Compute reference transpose solution
+  computeTransposeGold(transposeGold, h_idata, size_x, size_y);
+
+  // print out common data for all kernels
+  printf(
+      "\nMatrix size: %dx%d (%dx%d tiles), tile size: %dx%d, block size: "
+      "%dx%d\n\n",
+      size_x, size_y, size_x / TILE_DIM, size_y / TILE_DIM, TILE_DIM, TILE_DIM,
+      TILE_DIM, BLOCK_ROWS);
+
+  // initialize events
+  checkCudaErrors(cudaEventCreate(&start));
+  checkCudaErrors(cudaEventCreate(&stop));
+
+  //
+  // loop over different kernels
+  //
+
+  bool success = true;
+
+  for (int k = 0; k < 8; k++) {
+    // set kernel pointer
+    switch (k) {
+      case 0:
+        kernel = &copy;
+        kernelName = "simple copy       ";
+        break;
+
+      case 1:
+        kernel = &copySharedMem;
+        kernelName = "shared memory copy";
+        break;
+
+      case 2:
+        kernel = &transposeNaive;
+        kernelName = "naive             ";
+        break;
+
+      case 3:
+        kernel = &transposeCoalesced;
+        kernelName = "coalesced         ";
+        break;
+
+      case 4:
+        kernel = &transposeNoBankConflicts;
+        kernelName = "optimized         ";
+        break;
+
+      case 5:
+        kernel = &transposeCoarseGrained;
+        kernelName = "coarse-grained    ";
+        break;
+
+      case 6:
+        kernel = &transposeFineGrained;
+        kernelName = "fine-grained      ";
+        break;
+
+      case 7:
+        kernel = &transposeDiagonal;
+        kernelName = "diagonal          ";
+        break;
+    }
+
+    // set reference solution
+    if (kernel == &copy || kernel == &copySharedMem) {
+      gold = h_idata;
+    } else if (kernel == &transposeCoarseGrained ||
+               kernel == &transposeFineGrained) {
+      gold = h_odata;  // fine- and coarse-grained kernels are not full
+                       // transposes, so bypass check
+    } else {
+      gold = transposeGold;
+    }
+
+    // Clear error status
+    checkCudaErrors(cudaGetLastError());
+
+    // warmup to avoid timing startup
+    kernel<<<grid, threads>>>(d_odata, d_idata, size_x, size_y);
+
+    // take measurements for loop over kernel launches
+    checkCudaErrors(cudaEventRecord(start, 0));
+
+    for (int i = 0; i < NUM_REPS; i++) {
+      kernel<<<grid, threads>>>(d_odata, d_idata, size_x, size_y);
+      // Ensure no launch failure
+      checkCudaErrors(cudaGetLastError());
+    }
+
+    checkCudaErrors(cudaEventRecord(stop, 0));
+    checkCudaErrors(cudaEventSynchronize(stop));
+    float kernelTime;
+    checkCudaErrors(cudaEventElapsedTime(&kernelTime, start, stop));
+
+    checkCudaErrors(
+        cudaMemcpy(h_odata, d_odata, mem_size, cudaMemcpyDeviceToHost));
+    bool res = compareData(gold, h_odata, size_x * size_y, 0.01f, 0.0f);
+
+    if (res == false) {
+      printf("*** %s kernel FAILED ***\n", kernelName);
+      success = false;
+    }
+
+    // take measurements for loop inside kernel
+    checkCudaErrors(
+        cudaMemcpy(h_odata, d_odata, mem_size, cudaMemcpyDeviceToHost));
+    res = compareData(gold, h_odata, size_x * size_y, 0.01f, 0.0f);
+
+    if (res == false) {
+      printf("*** %s kernel FAILED ***\n", kernelName);
+      success = false;
+    }
+
+    // report effective bandwidths
+    float kernelBandwidth = 2.0f * 1000.0f * mem_size / (1024 * 1024 * 1024) /
+                            (kernelTime / NUM_REPS);
+    printf(
+        "transpose %s, Throughput = %.4f GB/s, Time = %.5f ms, Size = %u fp32 "
+        "elements, NumDevsUsed = %u, Workgroup = %u\n",
+        kernelName, kernelBandwidth, kernelTime / NUM_REPS, (size_x * size_y),
+        1, TILE_DIM * BLOCK_ROWS);
+  }
+
+  // cleanup
+  free(h_idata);
+  free(h_odata);
+  free(transposeGold);
+  cudaFree(d_idata);
+  cudaFree(d_odata);
+
+  checkCudaErrors(cudaEventDestroy(start));
+  checkCudaErrors(cudaEventDestroy(stop));
+
+  if (!success) {
+    printf("Test failed!\n");
+    exit(EXIT_FAILURE);
+  }
+
+  printf("Test passed\n");
+  exit(EXIT_SUCCESS);
+}

Samples/6_Performance/transpose/transpose_vs2017.sln

@@ -0,0 +1,20 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio 2017
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "transpose", "transpose_vs2017.vcxproj", "{997E0757-EA74-4A4E-A0FC-47D8C8831A15}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|x64 = Debug|x64
+		Release|x64 = Release|x64
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Debug|x64.ActiveCfg = Debug|x64
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Debug|x64.Build.0 = Debug|x64
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Release|x64.ActiveCfg = Release|x64
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Release|x64.Build.0 = Release|x64
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+EndGlobal

Samples/6_Performance/transpose/transpose_vs2017.vcxproj

@@ -0,0 +1,112 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <CUDAPropsPath Condition="'$(CUDAPropsPath)'==''">$(VCTargetsPath)\BuildCustomizations</CUDAPropsPath>
+  </PropertyGroup>
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Debug|x64">
+      <Configuration>Debug</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|x64">
+      <Configuration>Release</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>{997E0757-EA74-4A4E-A0FC-47D8C8831A15}</ProjectGuid>
+    <RootNamespace>transpose_vs2017</RootNamespace>
+    <ProjectName>transpose</ProjectName>
+    <CudaToolkitCustomDir />
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(WindowsTargetPlatformVersion)'==''">
+    <LatestTargetPlatformVersion>$([Microsoft.Build.Utilities.ToolLocationHelper]::GetLatestSDKTargetPlatformVersion('Windows', '10.0'))</LatestTargetPlatformVersion>
+    <WindowsTargetPlatformVersion Condition="'$(WindowsTargetPlatformVersion)' == ''">$(LatestTargetPlatformVersion)</WindowsTargetPlatformVersion>
+    <TargetPlatformVersion>$(WindowsTargetPlatformVersion)</TargetPlatformVersion>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <PropertyGroup>
+    <ConfigurationType>Application</ConfigurationType>
+    <CharacterSet>MultiByte</CharacterSet>
+    <PlatformToolset>v141</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)'=='Debug'">
+    <UseDebugLibraries>true</UseDebugLibraries>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)'=='Release'">
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <ImportGroup Label="ExtensionSettings">
+    <Import Project="$(CUDAPropsPath)\CUDA 11.6.props" />
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets">
+    <Import Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" />
+  </ImportGroup>
+  <PropertyGroup Label="UserMacros" />
+  <PropertyGroup>
+    <IntDir>$(Platform)/$(Configuration)/</IntDir>
+    <IncludePath>$(IncludePath)</IncludePath>
+    <CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRules />
+    <CodeAnalysisRuleAssemblies />
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Platform)'=='x64'">
+    <OutDir>../../../bin/win64/$(Configuration)/</OutDir>
+  </PropertyGroup>
+  <ItemDefinitionGroup>
+    <ClCompile>
+      <WarningLevel>Level3</WarningLevel>
+      <PreprocessorDefinitions>WIN32;_MBCS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <AdditionalIncludeDirectories>./;$(CudaToolkitDir)/include;../../../Common;</AdditionalIncludeDirectories>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <AdditionalDependencies>cudart_static.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalLibraryDirectories>$(CudaToolkitLibDir);</AdditionalLibraryDirectories>
+      <OutputFile>$(OutDir)/transpose.exe</OutputFile>
+    </Link>
+    <CudaCompile>
+      <CodeGeneration>compute_35,sm_35;compute_37,sm_37;compute_50,sm_50;compute_52,sm_52;compute_60,sm_60;compute_61,sm_61;compute_70,sm_70;compute_75,sm_75;compute_80,sm_80;compute_86,sm_86;</CodeGeneration>
+      <AdditionalOptions>-Xcompiler "/wd 4819"  --threads 0 </AdditionalOptions>
+      <Include>./;../../../Common</Include>
+      <Defines>WIN32</Defines>
+    </CudaCompile>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)'=='Debug'">
+    <ClCompile>
+      <Optimization>Disabled</Optimization>
+      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <LinkTimeCodeGeneration>Default</LinkTimeCodeGeneration>
+    </Link>
+    <CudaCompile>
+      <Runtime>MTd</Runtime>
+      <TargetMachinePlatform>64</TargetMachinePlatform>
+    </CudaCompile>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)'=='Release'">
+    <ClCompile>
+      <Optimization>MaxSpeed</Optimization>
+      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <GenerateDebugInformation>false</GenerateDebugInformation>
+      <LinkTimeCodeGeneration>UseLinkTimeCodeGeneration</LinkTimeCodeGeneration>
+    </Link>
+    <CudaCompile>
+      <Runtime>MT</Runtime>
+      <TargetMachinePlatform>64</TargetMachinePlatform>
+    </CudaCompile>
+  </ItemDefinitionGroup>
+  <ItemGroup>
+    <CudaCompile Include="transpose.cu" />
+
+  </ItemGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <ImportGroup Label="ExtensionTargets">
+    <Import Project="$(CUDAPropsPath)\CUDA 11.6.targets" />
+  </ImportGroup>
+</Project>

Samples/6_Performance/transpose/transpose_vs2019.sln

@@ -0,0 +1,20 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio 2019
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "transpose", "transpose_vs2019.vcxproj", "{997E0757-EA74-4A4E-A0FC-47D8C8831A15}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|x64 = Debug|x64
+		Release|x64 = Release|x64
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Debug|x64.ActiveCfg = Debug|x64
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Debug|x64.Build.0 = Debug|x64
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Release|x64.ActiveCfg = Release|x64
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Release|x64.Build.0 = Release|x64
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+EndGlobal

Samples/6_Performance/transpose/transpose_vs2019.vcxproj

@@ -0,0 +1,108 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <CUDAPropsPath Condition="'$(CUDAPropsPath)'==''">$(VCTargetsPath)\BuildCustomizations</CUDAPropsPath>
+  </PropertyGroup>
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Debug|x64">
+      <Configuration>Debug</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|x64">
+      <Configuration>Release</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>{997E0757-EA74-4A4E-A0FC-47D8C8831A15}</ProjectGuid>
+    <RootNamespace>transpose_vs2019</RootNamespace>
+    <ProjectName>transpose</ProjectName>
+    <CudaToolkitCustomDir />
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <PropertyGroup>
+    <ConfigurationType>Application</ConfigurationType>
+    <CharacterSet>MultiByte</CharacterSet>
+    <PlatformToolset>v142</PlatformToolset>
+    <WindowsTargetPlatformVersion>10.0</WindowsTargetPlatformVersion>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)'=='Debug'">
+    <UseDebugLibraries>true</UseDebugLibraries>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)'=='Release'">
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <ImportGroup Label="ExtensionSettings">
+    <Import Project="$(CUDAPropsPath)\CUDA 11.6.props" />
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets">
+    <Import Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" />
+  </ImportGroup>
+  <PropertyGroup Label="UserMacros" />
+  <PropertyGroup>
+    <IntDir>$(Platform)/$(Configuration)/</IntDir>
+    <IncludePath>$(IncludePath)</IncludePath>
+    <CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRules />
+    <CodeAnalysisRuleAssemblies />
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Platform)'=='x64'">
+    <OutDir>../../../bin/win64/$(Configuration)/</OutDir>
+  </PropertyGroup>
+  <ItemDefinitionGroup>
+    <ClCompile>
+      <WarningLevel>Level3</WarningLevel>
+      <PreprocessorDefinitions>WIN32;_MBCS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <AdditionalIncludeDirectories>./;$(CudaToolkitDir)/include;../../../Common;</AdditionalIncludeDirectories>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <AdditionalDependencies>cudart_static.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalLibraryDirectories>$(CudaToolkitLibDir);</AdditionalLibraryDirectories>
+      <OutputFile>$(OutDir)/transpose.exe</OutputFile>
+    </Link>
+    <CudaCompile>
+      <CodeGeneration>compute_35,sm_35;compute_37,sm_37;compute_50,sm_50;compute_52,sm_52;compute_60,sm_60;compute_61,sm_61;compute_70,sm_70;compute_75,sm_75;compute_80,sm_80;compute_86,sm_86;</CodeGeneration>
+      <AdditionalOptions>-Xcompiler "/wd 4819"  --threads 0 </AdditionalOptions>
+      <Include>./;../../../Common</Include>
+      <Defines>WIN32</Defines>
+    </CudaCompile>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)'=='Debug'">
+    <ClCompile>
+      <Optimization>Disabled</Optimization>
+      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <LinkTimeCodeGeneration>Default</LinkTimeCodeGeneration>
+    </Link>
+    <CudaCompile>
+      <Runtime>MTd</Runtime>
+      <TargetMachinePlatform>64</TargetMachinePlatform>
+    </CudaCompile>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)'=='Release'">
+    <ClCompile>
+      <Optimization>MaxSpeed</Optimization>
+      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <GenerateDebugInformation>false</GenerateDebugInformation>
+      <LinkTimeCodeGeneration>UseLinkTimeCodeGeneration</LinkTimeCodeGeneration>
+    </Link>
+    <CudaCompile>
+      <Runtime>MT</Runtime>
+      <TargetMachinePlatform>64</TargetMachinePlatform>
+    </CudaCompile>
+  </ItemDefinitionGroup>
+  <ItemGroup>
+    <CudaCompile Include="transpose.cu" />
+
+  </ItemGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <ImportGroup Label="ExtensionTargets">
+    <Import Project="$(CUDAPropsPath)\CUDA 11.6.targets" />
+  </ImportGroup>
+</Project>

Samples/6_Performance/transpose/transpose_vs2022.sln

@@ -0,0 +1,20 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio 2022
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "transpose", "transpose_vs2022.vcxproj", "{997E0757-EA74-4A4E-A0FC-47D8C8831A15}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|x64 = Debug|x64
+		Release|x64 = Release|x64
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Debug|x64.ActiveCfg = Debug|x64
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Debug|x64.Build.0 = Debug|x64
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Release|x64.ActiveCfg = Release|x64
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Release|x64.Build.0 = Release|x64
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+EndGlobal

Samples/6_Performance/transpose/transpose_vs2022.vcxproj

@@ -0,0 +1,108 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <CUDAPropsPath Condition="'$(CUDAPropsPath)'==''">$(VCTargetsPath)\BuildCustomizations</CUDAPropsPath>
+  </PropertyGroup>
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Debug|x64">
+      <Configuration>Debug</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|x64">
+      <Configuration>Release</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>{997E0757-EA74-4A4E-A0FC-47D8C8831A15}</ProjectGuid>
+    <RootNamespace>transpose_vs2022</RootNamespace>
+    <ProjectName>transpose</ProjectName>
+    <CudaToolkitCustomDir />
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <PropertyGroup>
+    <ConfigurationType>Application</ConfigurationType>
+    <CharacterSet>MultiByte</CharacterSet>
+    <PlatformToolset>v143</PlatformToolset>
+    <WindowsTargetPlatformVersion>10.0</WindowsTargetPlatformVersion>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)'=='Debug'">
+    <UseDebugLibraries>true</UseDebugLibraries>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)'=='Release'">
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <ImportGroup Label="ExtensionSettings">
+    <Import Project="$(CUDAPropsPath)\CUDA 11.6.props" />
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets">
+    <Import Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" />
+  </ImportGroup>
+  <PropertyGroup Label="UserMacros" />
+  <PropertyGroup>
+    <IntDir>$(Platform)/$(Configuration)/</IntDir>
+    <IncludePath>$(IncludePath)</IncludePath>
+    <CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRules />
+    <CodeAnalysisRuleAssemblies />
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Platform)'=='x64'">
+    <OutDir>../../../bin/win64/$(Configuration)/</OutDir>
+  </PropertyGroup>
+  <ItemDefinitionGroup>
+    <ClCompile>
+      <WarningLevel>Level3</WarningLevel>
+      <PreprocessorDefinitions>WIN32;_MBCS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <AdditionalIncludeDirectories>./;$(CudaToolkitDir)/include;../../../Common;</AdditionalIncludeDirectories>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <AdditionalDependencies>cudart_static.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalLibraryDirectories>$(CudaToolkitLibDir);</AdditionalLibraryDirectories>
+      <OutputFile>$(OutDir)/transpose.exe</OutputFile>
+    </Link>
+    <CudaCompile>
+      <CodeGeneration>compute_35,sm_35;compute_37,sm_37;compute_50,sm_50;compute_52,sm_52;compute_60,sm_60;compute_61,sm_61;compute_70,sm_70;compute_75,sm_75;compute_80,sm_80;compute_86,sm_86;</CodeGeneration>
+      <AdditionalOptions>-Xcompiler "/wd 4819"  --threads 0 </AdditionalOptions>
+      <Include>./;../../../Common</Include>
+      <Defines>WIN32</Defines>
+    </CudaCompile>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)'=='Debug'">
+    <ClCompile>
+      <Optimization>Disabled</Optimization>
+      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <LinkTimeCodeGeneration>Default</LinkTimeCodeGeneration>
+    </Link>
+    <CudaCompile>
+      <Runtime>MTd</Runtime>
+      <TargetMachinePlatform>64</TargetMachinePlatform>
+    </CudaCompile>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)'=='Release'">
+    <ClCompile>
+      <Optimization>MaxSpeed</Optimization>
+      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <GenerateDebugInformation>false</GenerateDebugInformation>
+      <LinkTimeCodeGeneration>UseLinkTimeCodeGeneration</LinkTimeCodeGeneration>
+    </Link>
+    <CudaCompile>
+      <Runtime>MT</Runtime>
+      <TargetMachinePlatform>64</TargetMachinePlatform>
+    </CudaCompile>
+  </ItemDefinitionGroup>
+  <ItemGroup>
+    <CudaCompile Include="transpose.cu" />
+
+  </ItemGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <ImportGroup Label="ExtensionTargets">
+    <Import Project="$(CUDAPropsPath)\CUDA 11.6.targets" />
+  </ImportGroup>
+</Project>