CUDA 13.2 samples update (#432)

- Added Python samples for CUDA Python 1.0 release
- Renamed top-level `Samples` directory to `cpp` to accommodate Python samples.

cpp/6_Performance/CMakeLists.txt

@@ -0,0 +1,5 @@
+add_subdirectory(LargeKernelParameter)
+add_subdirectory(UnifiedMemoryPerf)
+add_subdirectory(alignedTypes)
+add_subdirectory(cudaGraphsPerfScaling)
+add_subdirectory(transpose)

cpp/6_Performance/LargeKernelParameter/.vscode/c_cpp_properties.json

@@ -0,0 +1,18 @@
+{
+    "configurations": [
+        {
+            "name": "Linux",
+            "includePath": [
+                "${workspaceFolder}/**",
+                "${workspaceFolder}/../../../Common"
+            ],
+            "defines": [],
+            "compilerPath": "/usr/local/cuda/bin/nvcc",
+            "cStandard": "gnu17",
+            "cppStandard": "gnu++14",
+            "intelliSenseMode": "linux-gcc-x64",
+            "configurationProvider": "ms-vscode.makefile-tools"
+        }
+    ],
+    "version": 4
+}

cpp/6_Performance/LargeKernelParameter/.vscode/extensions.json

@@ -0,0 +1,7 @@
+{
+    "recommendations": [
+        "nvidia.nsight-vscode-edition",
+        "ms-vscode.cpptools",
+        "ms-vscode.makefile-tools"
+    ]
+}

cpp/6_Performance/LargeKernelParameter/CMakeLists.txt

@@ -0,0 +1,35 @@
+cmake_minimum_required(VERSION 3.20)
+
+list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/../../../cmake/Modules")
+
+project(LargeKernelParameter LANGUAGES C CXX CUDA)
+
+find_package(CUDAToolkit REQUIRED)
+
+set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+
+set(CMAKE_CUDA_ARCHITECTURES 75 80 86 87 89 90 100 110 120)
+set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Wno-deprecated-gpu-targets")
+
+if(ENABLE_CUDA_DEBUG)
+    set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -G")        # enable cuda-gdb (may significantly affect performance on some targets)
+else()
+    set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -lineinfo") # add line information to all builds for debug tools (exclusive to -G option)
+endif()
+
+# Include directories and libraries
+include_directories(../../../Common)
+
+# Source file
+# Add target for LargeKernelParameter
+add_executable(LargeKernelParameter LargeKernelParameter.cu)
+
+target_compile_options(LargeKernelParameter PRIVATE $<$<COMPILE_LANGUAGE:CUDA>:--extended-lambda>)
+
+target_compile_features(LargeKernelParameter PRIVATE cxx_std_17 cuda_std_17)
+
+set_target_properties(LargeKernelParameter PROPERTIES CUDA_SEPARABLE_COMPILATION ON)
+
+# Include installation configuration
+include(${CMAKE_CURRENT_SOURCE_DIR}/../../../cmake/InstallSamples.cmake)
+setup_samples_install()

cpp/6_Performance/LargeKernelParameter/LargeKernelParameter.cu

@@ -0,0 +1,180 @@
+/* Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * This is a simple test showing performance and usability
+ * improvements with large kernel parameters introduced in CUDA 12.1
+ */
+#include <cassert>
+#include <chrono>
+#include <iostream>
+
+// Utility includes
+#include <helper_cuda.h>
+
+using namespace std;
+using namespace std::chrono;
+
+#define TEST_ITERATIONS     (1000)
+#define TOTAL_PARAMS        (8000) // ints
+#define KERNEL_PARAM_LIMIT  (1024) // ints
+#define CONST_COPIED_PARAMS (TOTAL_PARAMS - KERNEL_PARAM_LIMIT)
+
+__constant__ int excess_params[CONST_COPIED_PARAMS];
+
+typedef struct
+{
+    int param[KERNEL_PARAM_LIMIT];
+} param_t;
+
+typedef struct
+{
+    int param[TOTAL_PARAMS];
+} param_large_t;
+
+// Kernel with 4KB kernel parameter limit
+__global__ void kernelDefault(__grid_constant__ const param_t p, int *result)
+{
+    int tmp = 0;
+
+    // accumulate kernel parameters
+    for (int i = 0; i < KERNEL_PARAM_LIMIT; ++i) {
+        tmp += p.param[i];
+    }
+
+    // accumulate excess values passed via const memory
+    for (int i = 0; i < CONST_COPIED_PARAMS; ++i) {
+        tmp += excess_params[i];
+    }
+
+    *result = tmp;
+}
+
+// Kernel with 32,764 byte kernel parameter limit
+__global__ void kernelLargeParam(__grid_constant__ const param_large_t p, int *result)
+{
+    int tmp = 0;
+
+    // accumulate kernel parameters
+    for (int i = 0; i < TOTAL_PARAMS; ++i) {
+        tmp += p.param[i];
+    }
+
+    *result = tmp;
+}
+
+static void report_time(std::chrono::time_point<std::chrono::steady_clock> start,
+                        std::chrono::time_point<std::chrono::steady_clock> end,
+                        int                                                iters)
+{
+    auto usecs = duration_cast<duration<float, microseconds::period>>(end - start);
+    cout << usecs.count() / iters << endl;
+}
+
+int main()
+{
+    int rc;
+    cudaFree(0);
+
+    param_t       p;
+    param_large_t p_large;
+
+    // pageable host memory that holds excess constants passed via constant memory
+    int *copied_params = (int *)malloc(CONST_COPIED_PARAMS * sizeof(int));
+    assert(copied_params);
+
+    // storage for computed result
+    int *d_result;
+    int  h_result;
+    checkCudaErrors(cudaMalloc(&d_result, sizeof(int)));
+
+    int expected_result = 0;
+
+    // fill in data for validation
+    for (int i = 0; i < KERNEL_PARAM_LIMIT; ++i) {
+        p.param[i] = (i & 0xFF);
+    }
+    for (int i = KERNEL_PARAM_LIMIT; i < TOTAL_PARAMS; ++i) {
+        copied_params[i - KERNEL_PARAM_LIMIT] = (i & 0xFF);
+    }
+    for (int i = 0; i < TOTAL_PARAMS; ++i) {
+        p_large.param[i] = (i & 0xFF);
+        expected_result += (i & 0xFF);
+    }
+
+    // warmup, verify correctness
+    checkCudaErrors(
+        cudaMemcpyToSymbol(excess_params, copied_params, CONST_COPIED_PARAMS * sizeof(int), 0, cudaMemcpyHostToDevice));
+    kernelDefault<<<1, 1>>>(p, d_result);
+    checkCudaErrors(cudaMemcpy(&h_result, d_result, sizeof(int), cudaMemcpyDeviceToHost));
+    checkCudaErrors(cudaDeviceSynchronize());
+    if (h_result != expected_result) {
+        std::cout << "Test failed" << std::endl;
+        rc = -1;
+        goto Exit;
+    }
+
+    kernelLargeParam<<<1, 1>>>(p_large, d_result);
+    checkCudaErrors(cudaMemcpy(&h_result, d_result, sizeof(int), cudaMemcpyDeviceToHost));
+    checkCudaErrors(cudaDeviceSynchronize());
+    if (h_result != expected_result) {
+        std::cout << "Test failed" << std::endl;
+        rc = -1;
+        goto Exit;
+    }
+
+    // benchmark default kernel parameter limit
+    {
+        auto start = steady_clock::now();
+        for (int i = 0; i < TEST_ITERATIONS; ++i) {
+            checkCudaErrors(cudaMemcpyToSymbol(
+                excess_params, copied_params, CONST_COPIED_PARAMS * sizeof(int), 0, cudaMemcpyHostToDevice));
+            kernelDefault<<<1, 1>>>(p, d_result);
+        }
+        checkCudaErrors(cudaDeviceSynchronize());
+        auto end = steady_clock::now();
+        std::cout << "Kernel 4KB parameter limit - time (us):";
+        report_time(start, end, TEST_ITERATIONS);
+
+        // benchmark large kernel parameter limit
+        start = steady_clock::now();
+        for (int i = 0; i < TEST_ITERATIONS; ++i) {
+            kernelLargeParam<<<1, 1>>>(p_large, d_result);
+        }
+        checkCudaErrors(cudaDeviceSynchronize());
+        end = steady_clock::now();
+        std::cout << "Kernel 32,764 byte parameter limit - time (us):";
+        report_time(start, end, TEST_ITERATIONS);
+    }
+    std::cout << "Test passed!" << std::endl;
+    rc = 0;
+Exit:
+    // cleanup
+    cudaFree(d_result);
+    free(copied_params);
+    return rc;
+}

cpp/6_Performance/LargeKernelParameter/README.md

@@ -0,0 +1,32 @@
+# LargeKernelParameter - Large Kernel Parameters
+
+## Description
+
+Simple test that demonstrates performance and usability improvements with large kernel parameters introduced in CUDA 12.1
+
+## Key Concepts
+
+Performance Strategies
+
+## Supported SM Architectures
+
+[SM 7.0 ](https://developer.nvidia.com/cuda-gpus)  [SM 7.2 ](https://developer.nvidia.com/cuda-gpus)  [SM 7.5 ](https://developer.nvidia.com/cuda-gpus)  [SM 8.0 ](https://developer.nvidia.com/cuda-gpus)  [SM 8.6 ](https://developer.nvidia.com/cuda-gpus)  [SM 8.7 ](https://developer.nvidia.com/cuda-gpus)  [SM 8.9 ](https://developer.nvidia.com/cuda-gpus)  [SM 9.0 ](https://developer.nvidia.com/cuda-gpus)
+
+## Supported OSes
+
+Linux, Windows
+
+## Supported CPU Architecture
+
+x86_64, armv7l
+
+## CUDA APIs involved
+
+### [CUDA Runtime API](http://docs.nvidia.com/cuda/cuda-runtime-api/index.html)
+cudaMemcpyToSymbol, cudaMemcpy, cudaFree, cudaDeviceSynchronize, cudaMalloc
+
+## Prerequisites
+
+Download and install the [CUDA Toolkit 12.1](https://developer.nvidia.com/cuda-downloads) for your corresponding platform.
+
+## References (for more details)

cpp/6_Performance/README.md

@@ -0,0 +1,14 @@
+# 6. Performance
+
+
+### [alignedTypes](./alignedTypes)
+A simple test, showing huge access speed gap between aligned and misaligned structures. It measures per-element copy throughput for aligned and misaligned structures on big chunks of data.
+
+### [transpose](./transpose)
+This sample demonstrates Matrix Transpose.  Different performance are shown to achieve high performance.
+
+### [UnifiedMemoryPerf](./UnifiedMemoryPerf)
+This sample demonstrates the performance comparision using matrix multiplication kernel of Unified Memory with/without hints and other types of memory like zero copy buffers, pageable, pagelocked memory performing synchronous and Asynchronous transfers on a single GPU.
+
+### [cudaGraphsPerfScaling](./cudaGraphsPerfScaling)
+This sample demonstrates the performance characteristics of cuda graphs. It is focused on how the apis scale with graph size.

cpp/6_Performance/UnifiedMemoryPerf/.vscode/c_cpp_properties.json

@@ -0,0 +1,18 @@
+{
+    "configurations": [
+        {
+            "name": "Linux",
+            "includePath": [
+                "${workspaceFolder}/**",
+                "${workspaceFolder}/../../../Common"
+            ],
+            "defines": [],
+            "compilerPath": "/usr/local/cuda/bin/nvcc",
+            "cStandard": "gnu17",
+            "cppStandard": "gnu++14",
+            "intelliSenseMode": "linux-gcc-x64",
+            "configurationProvider": "ms-vscode.makefile-tools"
+        }
+    ],
+    "version": 4
+}

cpp/6_Performance/UnifiedMemoryPerf/.vscode/extensions.json

@@ -0,0 +1,7 @@
+{
+    "recommendations": [
+        "nvidia.nsight-vscode-edition",
+        "ms-vscode.cpptools",
+        "ms-vscode.makefile-tools"
+    ]
+}

cpp/6_Performance/UnifiedMemoryPerf/CMakeLists.txt

@@ -0,0 +1,38 @@
+cmake_minimum_required(VERSION 3.20)
+
+list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/../../../cmake/Modules")
+
+project(UnifiedMemoryPerf LANGUAGES C CXX CUDA)
+
+find_package(CUDAToolkit REQUIRED)
+
+set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+
+set(CMAKE_CUDA_ARCHITECTURES 75 80 86 87 89 90 100 110 120)
+set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Wno-deprecated-gpu-targets")
+if(ENABLE_CUDA_DEBUG)
+    set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -G")        # enable cuda-gdb (may significantly affect performance on some targets)
+else()
+    set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -lineinfo") # add line information to all builds for debug tools (exclusive to -G option)
+endif()
+
+# Include directories and libraries
+include_directories(../../../Common)
+
+# Source file
+# Add target for UnifiedMemoryPerf
+add_executable(UnifiedMemoryPerf helperFunctions.cpp matrixMultiplyPerf.cu commonKernels.cu)
+
+target_compile_options(UnifiedMemoryPerf PRIVATE $<$<COMPILE_LANGUAGE:CUDA>:--extended-lambda>)
+
+target_compile_features(UnifiedMemoryPerf PRIVATE cxx_std_17 cuda_std_17)
+
+set_target_properties(UnifiedMemoryPerf PROPERTIES CUDA_SEPARABLE_COMPILATION ON)
+
+target_include_directories(UnifiedMemoryPerf PRIVATE
+    ${CUDAToolkit_INCLUDE_DIRS}
+)
+
+# Include installation configuration
+include(${CMAKE_CURRENT_SOURCE_DIR}/../../../cmake/InstallSamples.cmake)
+setup_samples_install()

cpp/6_Performance/UnifiedMemoryPerf/README.md

@@ -0,0 +1,34 @@
+# UnifiedMemoryPerf - Unified and other CUDA Memories Performance
+
+## Description
+
+This sample demonstrates the performance comparision using matrix multiplication kernel of Unified Memory with/without hints and other types of memory like zero copy buffers, pageable, pagelocked memory performing synchronous and Asynchronous transfers on a single GPU.
+
+## Key Concepts
+
+CUDA Systems Integration, Unified Memory, CUDA Streams and Events, Pinned System Paged Memory
+
+## Supported SM Architectures
+
+## Supported OSes
+
+Linux, Windows
+
+## Supported CPU Architecture
+
+x86_64, armv7l, aarch64
+
+## CUDA APIs involved
+
+### [CUDA Runtime API](http://docs.nvidia.com/cuda/cuda-runtime-api/index.html)
+cudaMemcpy, cudaStreamDestroy, cudaMemPrefetchAsync, cudaFree, cudaMallocHost, cudaMallocManaged, cudaStreamAttachMemAsync, cudaHostGetDevicePointer, cudaFreeHost, cudaStreamSynchronize, cudaMalloc, cudaMemcpyAsync, cudaStreamCreate, cudaGetDeviceProperties
+
+## Dependencies needed to build/run
+[UVM](../../../README.md#unified-virtual-memory)
+
+## Prerequisites
+
+Download and install the [CUDA Toolkit](https://developer.nvidia.com/cuda-downloads) for your corresponding platform.
+Make sure the dependencies mentioned in [Dependencies]() section above are installed.
+
+## References (for more details)

cpp/6_Performance/UnifiedMemoryPerf/commonDefs.hpp

@@ -0,0 +1,81 @@
+/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _COMMON_DEFS_
+#define _COMMON_DEFS_
+#include <cuda.h>
+
+#define ONE_KB 1024
+#define ONE_MB (ONE_KB * ONE_KB)
+
+extern size_t maxSampleSizeInMb;
+extern int    numKernelRuns;
+extern int    verboseResults;
+
+extern unsigned int findNumSizesToTest(unsigned int minSize, unsigned int maxSize, unsigned int multiplier);
+
+// For Tracking the different memory allocation types
+typedef enum memAllocType_enum {
+    MEMALLOC_TYPE_START,
+    USE_MANAGED_MEMORY_WITH_HINTS = MEMALLOC_TYPE_START,
+    USE_MANAGED_MEMORY_WITH_HINTS_ASYNC,
+    USE_MANAGED_MEMORY,
+    USE_ZERO_COPY,
+    USE_HOST_PAGEABLE_AND_DEVICE_MEMORY,
+    USE_HOST_PAGEABLE_AND_DEVICE_MEMORY_ASYNC,
+    USE_HOST_PAGELOCKED_AND_DEVICE_MEMORY,
+    USE_HOST_PAGELOCKED_AND_DEVICE_MEMORY_ASYNC,
+    MEMALLOC_TYPE_END = USE_HOST_PAGELOCKED_AND_DEVICE_MEMORY_ASYNC,
+    MEMALLOC_TYPE_INVALID,
+    MEMALLOC_TYPE_COUNT = MEMALLOC_TYPE_INVALID
+} MemAllocType;
+
+typedef enum bandwidthType_enum { READ_BANDWIDTH, WRITE_BANDWIDTH } BandwidthType;
+
+extern const char *memAllocTypeStr[];
+extern const char *memAllocTypeShortStr[];
+
+struct resultsData;
+struct testResults;
+
+void           createAndInitTestResults(struct testResults **results,
+                                        const char          *testName,
+                                        unsigned int         numMeasurements,
+                                        unsigned int         numSizesToTest);
+unsigned long *getPtrSizesToTest(struct testResults *results);
+
+void freeTestResultsAndAllResultsData(struct testResults *results);
+
+void    createResultDataAndAddToTestResults(struct resultsData **ptrData,
+                                            struct testResults  *results,
+                                            const char          *resultsName,
+                                            bool                 printOnlyInVerbose,
+                                            bool                 reportAsBandwidth);
+double *getPtrRunTimesInMs(struct resultsData *data, int allocType, int sizeIndex);
+
+void printResults(struct testResults *results, bool print_launch_transfer_results, bool print_std_deviation);
+#endif

cpp/6_Performance/UnifiedMemoryPerf/commonKernels.cu

@@ -0,0 +1,34 @@
+/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "commonKernels.hpp"
+
+__global__ void spinWhileLessThanOne(volatile unsigned int *latch)
+{
+    while (latch[0] < 1)
+        ;
+}

cpp/6_Performance/UnifiedMemoryPerf/commonKernels.hpp

@@ -0,0 +1,28 @@
+/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+__global__ void spinWhileLessThanOne(volatile unsigned int *latch);

cpp/6_Performance/UnifiedMemoryPerf/helperFunctions.cpp

@@ -0,0 +1,293 @@
+/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <stdio.h>
+#include <string.h>
+
+#include "commonDefs.hpp"
+#define CU_INIT_UUID
+#include <cmath>
+
+#define UNITS_Time "ms"
+#define UNITS_BW   "MB/s"
+#define KB_str     "KB"
+#define MB_str     "MB"
+
+struct resultsData
+{
+    char                resultsName[64];
+    struct testResults *results;
+    // this has MEMALLOC_TYPE_COUNT * results->numSizesToTest *
+    // results->numMeasurements elements
+    double            **runTimesInMs[MEMALLOC_TYPE_COUNT];
+    double             *averageRunTimesInMs[MEMALLOC_TYPE_COUNT];
+    double             *stdDevRunTimesInMs[MEMALLOC_TYPE_COUNT];
+    double             *stdDevBandwidthInMBps[MEMALLOC_TYPE_COUNT];
+    bool                printOnlyInVerbose;
+    bool                reportAsBandwidth;
+    struct resultsData *next;
+};
+
+struct testResults
+{
+    char                testName[64];
+    unsigned int        numMeasurements;
+    unsigned long      *sizesToTest;
+    unsigned int        numSizesToTest;
+    struct resultsData *resultsDataHead;
+    struct resultsData *resultsDataTail;
+};
+
+unsigned int findNumSizesToTest(unsigned int minSize, unsigned int maxSize, unsigned int multiplier)
+{
+    unsigned int numSizesToTest = 0;
+    while (minSize <= maxSize) {
+        numSizesToTest++;
+        minSize *= multiplier;
+    }
+    return numSizesToTest;
+}
+
+int compareDoubles(const void *ptr1, const void *ptr2) { return (*(double *)ptr1 > *(double *)ptr2) ? 1 : -1; }
+
+static inline double getTimeOrBandwidth(double runTimeInMs, unsigned long size, bool getBandwidth)
+{
+    return (getBandwidth) ? (1000 * (size / runTimeInMs)) / ONE_MB : runTimeInMs;
+}
+
+void createAndInitTestResults(struct testResults **ptrResults,
+                              const char          *testName,
+                              unsigned int         numMeasurements,
+                              unsigned int         numSizesToTest)
+{
+    unsigned int        i;
+    struct testResults *results;
+    results = (struct testResults *)malloc(sizeof(struct testResults));
+    memset(results, 0, sizeof(struct testResults));
+    strcpy(results->testName, testName);
+    results->numMeasurements = numMeasurements;
+    results->numSizesToTest  = numSizesToTest;
+    results->sizesToTest     = (unsigned long *)malloc(numSizesToTest * sizeof(unsigned long));
+    results->resultsDataHead = NULL;
+    results->resultsDataTail = NULL;
+
+    *ptrResults = results;
+}
+
+unsigned long *getPtrSizesToTest(struct testResults *results) { return results->sizesToTest; }
+
+void createResultDataAndAddToTestResults(struct resultsData **ptrData,
+                                         struct testResults  *results,
+                                         const char          *resultsName,
+                                         bool                 printOnlyInVerbose,
+                                         bool                 reportAsBandwidth)
+{
+    unsigned int        i, j;
+    struct resultsData *data;
+    data = (struct resultsData *)malloc(sizeof(struct resultsData));
+    memset(data, 0, sizeof(struct resultsData));
+    strcpy(data->resultsName, resultsName);
+    data->results = results;
+    for (i = 0; i < MEMALLOC_TYPE_COUNT; i++) {
+        data->runTimesInMs[i] = (double **)malloc(results->numSizesToTest * sizeof(double *));
+        for (j = 0; j < results->numSizesToTest; j++) {
+            data->runTimesInMs[i][j] = (double *)malloc(results->numMeasurements * sizeof(double));
+        }
+        data->averageRunTimesInMs[i]   = (double *)malloc(results->numSizesToTest * sizeof(double));
+        data->stdDevRunTimesInMs[i]    = (double *)malloc(results->numSizesToTest * sizeof(double));
+        data->stdDevBandwidthInMBps[i] = (double *)malloc(results->numSizesToTest * sizeof(double));
+    }
+    data->printOnlyInVerbose = printOnlyInVerbose;
+    data->reportAsBandwidth  = reportAsBandwidth;
+    data->next               = NULL;
+    *ptrData                 = data;
+    if (results->resultsDataHead == NULL) {
+        results->resultsDataHead = data;
+        results->resultsDataTail = data;
+    }
+    else {
+        results->resultsDataTail->next = data;
+        results->resultsDataTail       = data;
+    }
+}
+
+double *getPtrRunTimesInMs(struct resultsData *data, int allocType, int sizeIndex)
+{
+    return data->runTimesInMs[allocType][sizeIndex];
+}
+
+void freeTestResultsAndAllResultsData(struct testResults *results)
+{
+    struct resultsData *data, *dataToFree;
+    unsigned int        i, j;
+    for (data = results->resultsDataHead; data != NULL;) {
+        for (i = 0; i < MEMALLOC_TYPE_COUNT; i++) {
+            for (j = 0; j < results->numSizesToTest; j++) {
+                free(data->runTimesInMs[i][j]);
+            }
+            free(data->runTimesInMs[i]);
+            free(data->averageRunTimesInMs[i]);
+            free(data->stdDevRunTimesInMs[i]);
+            free(data->stdDevBandwidthInMBps[i]);
+        }
+        dataToFree = data;
+        data       = data->next;
+        free(dataToFree);
+    }
+    free(results->sizesToTest);
+    free(results);
+}
+
+void calculateAverageAndStdDev(double *pAverage, double *pStdDev, double *allResults, unsigned int count)
+{
+    unsigned int i;
+    double       average = 0.0;
+    double       stdDev  = 0.0;
+    for (i = 0; i < count; i++) {
+        average += allResults[i];
+    }
+    average /= count;
+    for (i = 0; i < count; i++) {
+        stdDev += (allResults[i] - average) * (allResults[i] - average);
+    }
+    stdDev /= count;
+    stdDev    = sqrt(stdDev);
+    *pAverage = average;
+    *pStdDev  = (average == 0.0) ? 0.0 : ((100.0 * stdDev) / average);
+}
+
+void calculateStdDevBandwidth(double *pStdDev, double *allResults, unsigned int count, unsigned long size)
+{
+    unsigned int i;
+    double       bandwidth;
+    double       average = 0.0;
+    double       stdDev  = 0.0;
+    for (i = 0; i < count; i++) {
+        bandwidth = (1000 * (size / allResults[i])) / ONE_MB;
+        average += bandwidth;
+    }
+    average /= count;
+    for (i = 0; i < count; i++) {
+        bandwidth = (1000 * (size / allResults[i])) / ONE_MB;
+        stdDev += (bandwidth - average) * (bandwidth - average);
+    }
+    stdDev /= count;
+    stdDev   = sqrt(stdDev);
+    *pStdDev = (average == 0.0) ? 0.0 : ((100.0 * stdDev) / average);
+}
+
+void printTimesInTableFormat(struct testResults *results, struct resultsData *data, bool printAverage, bool printStdDev)
+{
+    unsigned int i, j;
+    bool         printStdDevBandwidth = printStdDev && data->reportAsBandwidth;
+    printf("Size_KB");
+    for (i = 0; i < MEMALLOC_TYPE_COUNT; i++) {
+        printf("\t%7s", memAllocTypeShortStr[i]);
+    }
+    printf("\n");
+    for (j = 0; j < results->numSizesToTest; j++) {
+        printf("%lu", results->sizesToTest[j] / ONE_KB);
+        for (i = 0; i < MEMALLOC_TYPE_COUNT; i++) {
+            printf(data->reportAsBandwidth ? "\t%7.2lf" : "\t%7.3lf",
+                   printStdDevBandwidth ? data->stdDevBandwidthInMBps[i][j]
+                                        : getTimeOrBandwidth(printAverage ? data->averageRunTimesInMs[i][j]
+                                                                          : data->stdDevRunTimesInMs[i][j],
+                                                             results->sizesToTest[j],
+                                                             data->reportAsBandwidth));
+        }
+        printf("\n");
+    }
+}
+
+void printAllResultsInVerboseMode(struct testResults *results, struct resultsData *data)
+{
+    unsigned int i, j, k;
+    for (i = 0; i < MEMALLOC_TYPE_COUNT; i++) {
+        printf("Verbose mode, printing all results for %s\n", memAllocTypeStr[i]);
+        printf("Instance");
+        for (j = 0; j < results->numSizesToTest; j++) {
+            printf("\t%lu", results->sizesToTest[j] / ONE_KB);
+        }
+        printf("\n");
+        for (k = 0; k < results->numMeasurements; k++) {
+            printf("%u", k);
+            for (j = 0; j < results->numSizesToTest; j++) {
+                printf(
+                    data->reportAsBandwidth ? "\t%7.2lf" : "\t%7.3lf",
+                    getTimeOrBandwidth(data->runTimesInMs[i][j][k], results->sizesToTest[j], data->reportAsBandwidth));
+            }
+            printf("\n");
+        }
+    }
+}
+
+void printResults(struct testResults *results, bool print_launch_transfer_results, bool print_std_deviation)
+{
+    char                vulcanPrint[256];
+    char                resultNameNoSpaces[64];
+    unsigned int        i, j, k;
+    struct resultsData *resultsIter;
+    bool                sizeGreaterThan1MB;
+    for (resultsIter = results->resultsDataHead; resultsIter != NULL; resultsIter = resultsIter->next) {
+        if (!verboseResults && resultsIter->printOnlyInVerbose) {
+            continue;
+        }
+        if (!print_launch_transfer_results) {
+            if (!(strcmp(resultsIter->resultsName, "Overall Time") == 0)) {
+                continue;
+            }
+        }
+        // regular print
+        printf("\n%s For %s ", resultsIter->resultsName, results->testName);
+        printf("\n");
+        for (j = 0; j < results->numSizesToTest; j++) {
+            for (i = 0; i < MEMALLOC_TYPE_COUNT; i++) {
+                calculateAverageAndStdDev(&resultsIter->averageRunTimesInMs[i][j],
+                                          &resultsIter->stdDevRunTimesInMs[i][j],
+                                          resultsIter->runTimesInMs[i][j],
+                                          results->numMeasurements);
+                if (resultsIter->reportAsBandwidth) {
+                    calculateStdDevBandwidth(&resultsIter->stdDevBandwidthInMBps[i][j],
+                                             resultsIter->runTimesInMs[i][j],
+                                             results->numMeasurements,
+                                             results->sizesToTest[j]);
+                }
+            }
+        }
+        printf("\nPrinting Average of %u measurements in (%s)\n",
+               results->numMeasurements,
+               resultsIter->reportAsBandwidth ? UNITS_BW : UNITS_Time);
+        printTimesInTableFormat(results, resultsIter, true, false);
+        if (print_std_deviation) {
+            printf("\nPrinting Standard Deviation as %% of average of %u measurements\n", results->numMeasurements);
+            printTimesInTableFormat(results, resultsIter, false, true);
+        }
+        if (verboseResults) {
+            printAllResultsInVerboseMode(results, resultsIter);
+        }
+    }
+}

cpp/6_Performance/UnifiedMemoryPerf/matrixMultiplyPerf.cu

@@ -0,0 +1,684 @@
+/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <helper_cuda.h>
+#include <helper_timer.h>
+
+#include "commonDefs.hpp"
+#include "commonKernels.hpp"
+
+#define VERIFY_GPU_CORRECTNESS 0
+
+size_t maxSampleSizeInMb = 64;
+int    numKernelRuns     = 20;
+int    verboseResults    = 0;
+
+const char *memAllocTypeStr[MEMALLOC_TYPE_COUNT] = {"Managed_Memory_With_Hints",
+                                                    "Managed_Memory_With_Hints_FullyAsync",
+                                                    "Managed_Memory_NoHints",
+                                                    "Zero_Copy",
+                                                    "Memcpy_HostMalloc_DeviceCudaMalloc",
+                                                    "MemcpyAsync_HostMalloc_DeviceCudaMalloc",
+                                                    "Memcpy_HostCudaHostAlloc_DeviceCudaMalloc",
+                                                    "MemcpyAsync_HostCudaHostAlloc_DeviceCudaMalloc"};
+
+const char *memAllocTypeShortStr[MEMALLOC_TYPE_COUNT] = {
+    "UMhint",  // Managed Memory With Hints
+    "UMhntAs", // Managed Memory With_Hints Async
+    "UMeasy",  // Managed_Memory with No Hints
+    "0Copy",   // Zero Copy
+    "MemCopy", // USE HOST PAGEABLE AND DEVICE_MEMORY
+    "CpAsync", // USE HOST PAGEABLE AND DEVICE_MEMORY ASYNC
+    "CpHpglk", // USE HOST PAGELOCKED AND DEVICE MEMORY
+    "CpPglAs"  // USE HOST PAGELOCKED AND DEVICE MEMORY ASYNC
+};
+
+static float RandFloat(float low, float high)
+{
+    float t = (float)rand() / (float)RAND_MAX;
+    return (1.0f - t) * low + t * high;
+}
+
+void fillMatrixWithRandomValues(float *matrix, unsigned int matrixDim)
+{
+    unsigned int i, j;
+    for (i = 0; i < matrixDim; ++i) {
+        for (j = 0; j < matrixDim; ++j) {
+            matrix[j + i * matrixDim] = RandFloat(0.0f, 10.0f);
+        }
+    }
+}
+
+#if VERIFY_GPU_CORRECTNESS
+void verifyMatrixMultiplyCorrectness(float *C, float *A, float *B, unsigned int matrixDim)
+{
+    unsigned int i, j, k, numErrors = 0;
+    for (i = 0; i < matrixDim; ++i) {
+        for (j = 0; j < matrixDim; ++j) {
+            float result = 0.0f;
+            for (k = 0; k < matrixDim; ++k) {
+                result += A[k + i * matrixDim] * B[j + k * matrixDim];
+            }
+            if (fabs(C[j + i * matrixDim] - result) > 0.001 * matrixDim) {
+                printf("At [%u, %u]: Expected %f, Found %f\n", i, j, result, C[j + i * matrixDim]);
+                ++numErrors;
+            }
+        }
+    }
+    if (numErrors != 0) {
+        printf("%d value mismatches occured\n", numErrors);
+        fflush(stdout);
+        exit(EXIT_FAILURE); // exit since value mismatches occured
+    }
+}
+#endif
+
+void copyMatrix(float *dstMatrix, float *srcMatrix, unsigned int matrixDim)
+{
+    size_t size = matrixDim * matrixDim * sizeof(float);
+    memcpy(dstMatrix, srcMatrix, size);
+}
+
+void verifyMatrixData(float *expectedData, float *observedData, unsigned int matrixDim)
+{
+    unsigned int i, j, numErrors = 0;
+    for (i = 0; i < matrixDim; ++i) {
+        for (j = 0; j < matrixDim; ++j) {
+            if (expectedData[j + i * matrixDim] != observedData[j + i * matrixDim]) {
+                ++numErrors;
+                if (verboseResults) {
+                    printf("At [%u, %u]: Expected %f, Found %f\n",
+                           i,
+                           j,
+                           expectedData[j + i * matrixDim],
+                           observedData[j + i * matrixDim]);
+                }
+            }
+        }
+    }
+    if (numErrors != 0) {
+        printf("%d value mismatches occured\n", numErrors);
+        fflush(stdout);
+        exit(EXIT_FAILURE); // exit since value mismatches occured
+    }
+}
+
+#define BLOCK_SIZE 32
+__global__ void matrixMultiplyKernel(float *C, float *A, float *B, unsigned int matrixDim)
+{
+    // Block index
+    int bx = blockIdx.x;
+    int by = blockIdx.y;
+
+    // Thread index
+    int tx = threadIdx.x;
+    int ty = threadIdx.y;
+
+    unsigned int wA = matrixDim;
+    unsigned int wB = matrixDim;
+
+    // Index of the first sub-matrix of A processed by the block
+    int aBegin = matrixDim * BLOCK_SIZE * by;
+
+    // Index of the last sub-matrix of A processed by the block
+    int aEnd = aBegin + wA - 1;
+
+    // Step size used to iterate through the sub-matrices of A
+    int aStep = BLOCK_SIZE;
+
+    // Index of the first sub-matrix of B processed by the block
+    int bBegin = BLOCK_SIZE * bx;
+
+    // Step size used to iterate through the sub-matrices of B
+    int bStep = BLOCK_SIZE * wB;
+
+    // Csub is used to store the element of the block sub-matrix
+    // that is computed by the thread
+    float Csub = 0;
+
+    // Loop over all the sub-matrices of A and B
+    // required to compute the block sub-matrix
+    for (int a = aBegin, b = bBegin; a <= aEnd; a += aStep, b += bStep) {
+        // Declaration of the shared memory array As used to
+        // store the sub-matrix of A
+        __shared__ float As[BLOCK_SIZE][BLOCK_SIZE];
+
+        // Declaration of the shared memory array Bs used to
+        // store the sub-matrix of B
+        __shared__ float Bs[BLOCK_SIZE][BLOCK_SIZE];
+
+        // Load the matrices from device memory
+        // to shared memory; each thread loads
+        // one element of each matrix
+        As[ty][tx] = A[a + wA * ty + tx];
+        Bs[ty][tx] = B[b + wB * ty + tx];
+
+        // Synchronize to make sure the matrices are loaded
+        __syncthreads();
+
+        // Multiply the two matrices together;
+        // each thread computes one element
+        // of the block sub-matrix
+#pragma unroll
+
+        for (int k = 0; k < BLOCK_SIZE; ++k) {
+            Csub += As[ty][k] * Bs[k][tx];
+        }
+
+        // Synchronize to make sure that the preceding
+        // computation is done before loading two new
+        // sub-matrices of A and B in the next iteration
+        __syncthreads();
+    }
+
+    // Write the block sub-matrix to device memory;
+    // each thread writes one element
+    int c               = wB * BLOCK_SIZE * by + BLOCK_SIZE * bx;
+    C[c + wB * ty + tx] = Csub;
+}
+
+void runMatrixMultiplyKernel(unsigned int matrixDim,
+                             int          allocType,
+                             unsigned int numLoops,
+                             double      *gpuLaunchCallsTimes,
+                             double      *gpuTransferToCallsTimes,
+                             double      *gpuTransferFromCallsTimes,
+                             double      *gpuLaunchAndTransferCallsTimes,
+                             double      *gpuLaunchTransferSyncTimes,
+                             double      *cpuAccessTimes,
+                             double      *overallTimes,
+                             int          device_id)
+{
+    float              *dptrA = NULL, *hptrA = NULL;
+    float              *dptrB = NULL, *hptrB = NULL;
+    float              *dptrC = NULL, *hptrC = NULL;
+    float              *randValuesX = NULL, *randValuesY = NULL;
+    float              *randValuesVerifyXmulY = NULL, *randValuesVerifyYmulX = NULL;
+    bool                copyRequired = false, hintsRequired = false;
+    bool                someTransferOpRequired;
+    bool                isAsync = false;
+    cudaStream_t        streamToRunOn;
+    unsigned int       *latch;
+    size_t              size = matrixDim * matrixDim * sizeof(float);
+    dim3                threads(32, 32);
+    dim3                grid(matrixDim / threads.x, matrixDim / threads.y);
+    StopWatchInterface *gpuLaunchCallsTimer = 0, *gpuTransferCallsTimer = 0;
+    StopWatchInterface *gpuSyncTimer = 0, *cpuAccessTimer = 0;
+    sdkCreateTimer(&gpuLaunchCallsTimer);
+    sdkCreateTimer(&gpuTransferCallsTimer);
+    sdkCreateTimer(&gpuSyncTimer);
+    sdkCreateTimer(&cpuAccessTimer);
+    unsigned int i;
+
+    cudaDeviceProp deviceProp;
+    checkCudaErrors(cudaGetDeviceProperties(&deviceProp, device_id));
+    checkCudaErrors(cudaStreamCreate(&streamToRunOn));
+
+    randValuesX = (float *)malloc(size);
+    if (!randValuesX) {
+        exit(EXIT_FAILURE); // exit since memory allocation error
+    }
+    randValuesY = (float *)malloc(size);
+    if (!randValuesY) {
+        exit(EXIT_FAILURE); // exit since memory allocation error
+    }
+    randValuesVerifyXmulY = (float *)malloc(size);
+    if (!randValuesVerifyXmulY) {
+        exit(EXIT_FAILURE); // exit since memory allocation error
+    }
+    randValuesVerifyYmulX = (float *)malloc(size);
+    if (!randValuesVerifyYmulX) {
+        exit(EXIT_FAILURE); // exit since memory allocation error
+    }
+    checkCudaErrors(cudaMalloc(&dptrA, size));
+    checkCudaErrors(cudaMalloc(&dptrB, size));
+    checkCudaErrors(cudaMalloc(&dptrC, size));
+
+    fillMatrixWithRandomValues(randValuesX, matrixDim);
+    fillMatrixWithRandomValues(randValuesY, matrixDim);
+
+    checkCudaErrors(cudaMemcpyAsync(dptrA, randValuesX, size, cudaMemcpyHostToDevice));
+    checkCudaErrors(cudaMemcpyAsync(dptrB, randValuesY, size, cudaMemcpyHostToDevice));
+    matrixMultiplyKernel<<<grid, threads>>>(dptrC, dptrA, dptrB, matrixDim);
+    checkCudaErrors(cudaMemcpyAsync(randValuesVerifyXmulY, dptrC, size, cudaMemcpyDeviceToHost));
+    checkCudaErrors(cudaStreamSynchronize(NULL));
+    matrixMultiplyKernel<<<grid, threads>>>(dptrC, dptrB, dptrA, matrixDim);
+    checkCudaErrors(cudaMemcpyAsync(randValuesVerifyYmulX, dptrC, size, cudaMemcpyDeviceToHost));
+    checkCudaErrors(cudaStreamSynchronize(NULL));
+#if VERIFY_GPU_CORRECTNESS
+    verifyMatrixMultiplyCorrectness(randValuesVerifyXmulY, randValuesX, randValuesY, matrixDim);
+    verifyMatrixMultiplyCorrectness(randValuesVerifyYmulX, randValuesY, randValuesX, matrixDim);
+#endif
+    checkCudaErrors(cudaFree(dptrA));
+    checkCudaErrors(cudaFree(dptrB));
+    checkCudaErrors(cudaFree(dptrC));
+
+    checkCudaErrors(cudaMallocHost(&latch, sizeof(unsigned int)));
+
+    switch (allocType) {
+    case USE_HOST_PAGEABLE_AND_DEVICE_MEMORY:
+    case USE_HOST_PAGEABLE_AND_DEVICE_MEMORY_ASYNC:
+        hptrA = (float *)malloc(size);
+        if (!hptrA) {
+            exit(EXIT_FAILURE); // exit since memory allocation error
+        }
+        hptrB = (float *)malloc(size);
+        if (!hptrB) {
+            exit(EXIT_FAILURE); // exit since memory allocation error
+        }
+        hptrC = (float *)malloc(size);
+        if (!hptrC) {
+            exit(EXIT_FAILURE); // exit since memory allocation error
+        }
+        checkCudaErrors(cudaMalloc(&dptrA, size));
+        checkCudaErrors(cudaMalloc(&dptrB, size));
+        checkCudaErrors(cudaMalloc(&dptrC, size));
+        copyRequired = true;
+        break;
+
+    case USE_HOST_PAGELOCKED_AND_DEVICE_MEMORY:
+    case USE_HOST_PAGELOCKED_AND_DEVICE_MEMORY_ASYNC:
+        checkCudaErrors(cudaMallocHost(&hptrA, size));
+        checkCudaErrors(cudaMallocHost(&hptrB, size));
+        checkCudaErrors(cudaMallocHost(&hptrC, size));
+        checkCudaErrors(cudaMalloc(&dptrA, size));
+        checkCudaErrors(cudaMalloc(&dptrB, size));
+        checkCudaErrors(cudaMalloc(&dptrC, size));
+        copyRequired = true;
+        break;
+
+    case USE_ZERO_COPY:
+        checkCudaErrors(cudaMallocHost(&hptrA, size));
+        checkCudaErrors(cudaMallocHost(&hptrB, size));
+        checkCudaErrors(cudaMallocHost(&hptrC, size));
+        checkCudaErrors(cudaHostGetDevicePointer(&dptrA, hptrA, 0));
+        checkCudaErrors(cudaHostGetDevicePointer(&dptrB, hptrB, 0));
+        checkCudaErrors(cudaHostGetDevicePointer(&dptrC, hptrC, 0));
+        break;
+
+    case USE_MANAGED_MEMORY:
+        checkCudaErrors(cudaMallocManaged(&dptrA, size));
+        checkCudaErrors(cudaMallocManaged(&dptrB, size));
+        checkCudaErrors(cudaMallocManaged(&dptrC, size));
+        hptrA = dptrA;
+        hptrB = dptrB;
+        hptrC = dptrC;
+        break;
+
+    case USE_MANAGED_MEMORY_WITH_HINTS:
+    case USE_MANAGED_MEMORY_WITH_HINTS_ASYNC:
+        if (deviceProp.concurrentManagedAccess) {
+            checkCudaErrors(cudaMallocManaged(&dptrA, size));
+            checkCudaErrors(cudaMallocManaged(&dptrB, size));
+            checkCudaErrors(cudaMallocManaged(&dptrC, size));
+            cudaMemLocation hostLoc;
+            hostLoc.type = cudaMemLocationTypeHost;
+            checkCudaErrors(cudaMemPrefetchAsync(dptrA, size, hostLoc, 0));
+            checkCudaErrors(cudaMemPrefetchAsync(dptrB, size, hostLoc, 0));
+            checkCudaErrors(cudaMemPrefetchAsync(dptrC, size, hostLoc, 0));
+        }
+        else {
+            checkCudaErrors(cudaMallocManaged(&dptrA, size, cudaMemAttachHost));
+            checkCudaErrors(cudaMallocManaged(&dptrB, size, cudaMemAttachHost));
+            checkCudaErrors(cudaMallocManaged(&dptrC, size, cudaMemAttachHost));
+        }
+        hptrA         = dptrA;
+        hptrB         = dptrB;
+        hptrC         = dptrC;
+        hintsRequired = true;
+        break;
+
+    default:
+        exit(EXIT_FAILURE); // exit with error
+    }
+
+    if (allocType == USE_HOST_PAGEABLE_AND_DEVICE_MEMORY_ASYNC
+        || allocType == USE_HOST_PAGELOCKED_AND_DEVICE_MEMORY_ASYNC
+        || allocType == USE_MANAGED_MEMORY_WITH_HINTS_ASYNC) {
+        isAsync = true;
+    }
+
+    someTransferOpRequired = copyRequired || hintsRequired;
+
+    // fill buffers with 0 to avoid any first access page-fault overheads.
+    memset(hptrA, 0, size);
+    memset(hptrB, 0, size);
+    memset(hptrC, 0, size);
+
+    for (i = 0; i < numLoops; i++) {
+        cpuAccessTimes[i]            = 0.0;
+        gpuLaunchCallsTimes[i]       = 0.0;
+        gpuTransferToCallsTimes[i]   = 0.0;
+        gpuTransferFromCallsTimes[i] = 0.0;
+
+        sdkStartTimer(&cpuAccessTimer);
+        {
+            copyMatrix(hptrA, (i & 0x1 == 0) ? randValuesX : randValuesY, matrixDim);
+            copyMatrix(hptrB, (i & 0x1 == 0) ? randValuesY : randValuesX, matrixDim);
+        }
+        sdkStopTimer(&cpuAccessTimer);
+        cpuAccessTimes[i] += sdkGetAverageTimerValue(&cpuAccessTimer);
+        sdkResetTimer(&cpuAccessTimer);
+
+        if (isAsync && hintsRequired) {
+            *latch = 0;
+            // Prevent any work on stream from starting until all work is pushed
+            spinWhileLessThanOne<<<1, 1, 0, streamToRunOn>>>(latch);
+        }
+
+        if (someTransferOpRequired) {
+            sdkStartTimer(&gpuTransferCallsTimer);
+            if (copyRequired) {
+                if (isAsync) {
+                    checkCudaErrors(cudaMemcpyAsync(dptrA, hptrA, size, cudaMemcpyHostToDevice, streamToRunOn));
+                    checkCudaErrors(cudaMemcpyAsync(dptrB, hptrB, size, cudaMemcpyHostToDevice, streamToRunOn));
+                }
+                else {
+                    checkCudaErrors(cudaMemcpy(dptrA, hptrA, size, cudaMemcpyHostToDevice));
+                    checkCudaErrors(cudaMemcpy(dptrB, hptrB, size, cudaMemcpyHostToDevice));
+                }
+            }
+            if (hintsRequired) {
+                if (deviceProp.concurrentManagedAccess) {
+                    cudaMemLocation deviceLoc;
+                    deviceLoc.type = cudaMemLocationTypeDevice;
+                    deviceLoc.id   = device_id;
+                    checkCudaErrors(cudaMemPrefetchAsync(dptrA, size, deviceLoc, 0, streamToRunOn));
+                    checkCudaErrors(cudaMemPrefetchAsync(dptrB, size, deviceLoc, 0, streamToRunOn));
+                    checkCudaErrors(cudaMemPrefetchAsync(dptrC, size, deviceLoc, 0, streamToRunOn));
+                }
+                else {
+                    checkCudaErrors(cudaStreamAttachMemAsync(streamToRunOn, dptrA, 0, cudaMemAttachGlobal));
+                    checkCudaErrors(cudaStreamAttachMemAsync(streamToRunOn, dptrB, 0, cudaMemAttachGlobal));
+                    checkCudaErrors(cudaStreamAttachMemAsync(streamToRunOn, dptrC, 0, cudaMemAttachGlobal));
+                }
+                if (!isAsync) {
+                    checkCudaErrors(cudaStreamSynchronize(streamToRunOn));
+                }
+            }
+
+            sdkStopTimer(&gpuTransferCallsTimer);
+            gpuTransferToCallsTimes[i] += sdkGetAverageTimerValue(&gpuTransferCallsTimer);
+            sdkResetTimer(&gpuTransferCallsTimer);
+        }
+
+        sdkStartTimer(&gpuLaunchCallsTimer);
+        {
+            matrixMultiplyKernel<<<grid, threads, 0, streamToRunOn>>>(dptrC, dptrA, dptrB, matrixDim);
+            if (!isAsync) {
+                checkCudaErrors(cudaStreamSynchronize(streamToRunOn));
+            }
+        }
+        sdkStopTimer(&gpuLaunchCallsTimer);
+
+        gpuLaunchCallsTimes[i] += sdkGetAverageTimerValue(&gpuLaunchCallsTimer);
+        sdkResetTimer(&gpuLaunchCallsTimer);
+
+        if (someTransferOpRequired) {
+            sdkStartTimer(&gpuTransferCallsTimer);
+            if (hintsRequired) {
+                if (deviceProp.concurrentManagedAccess) {
+                    cudaMemLocation hostLoc;
+                    hostLoc.type = cudaMemLocationTypeHost;
+                    checkCudaErrors(cudaMemPrefetchAsync(dptrA, size, hostLoc, 0));
+                    checkCudaErrors(cudaMemPrefetchAsync(dptrB, size, hostLoc, 0));
+                    checkCudaErrors(cudaMemPrefetchAsync(dptrC, size, hostLoc, 0));
+                }
+                else {
+                    checkCudaErrors(cudaStreamAttachMemAsync(streamToRunOn, dptrA, 0, cudaMemAttachHost));
+                    checkCudaErrors(cudaStreamAttachMemAsync(streamToRunOn, dptrB, 0, cudaMemAttachHost));
+                    checkCudaErrors(cudaStreamAttachMemAsync(streamToRunOn, dptrC, 0, cudaMemAttachHost));
+                }
+                if (!isAsync) {
+                    checkCudaErrors(cudaStreamSynchronize(streamToRunOn));
+                }
+            }
+            if (copyRequired) {
+                if (isAsync) {
+                    checkCudaErrors(cudaMemcpyAsync(hptrC, dptrC, size, cudaMemcpyDeviceToHost, streamToRunOn));
+                }
+                else {
+                    checkCudaErrors(cudaMemcpy(hptrC, dptrC, size, cudaMemcpyDeviceToHost));
+                }
+            }
+            sdkStopTimer(&gpuTransferCallsTimer);
+            gpuTransferFromCallsTimes[i] += sdkGetAverageTimerValue(&gpuTransferCallsTimer);
+            sdkResetTimer(&gpuTransferCallsTimer);
+        }
+        gpuLaunchAndTransferCallsTimes[i] =
+            gpuLaunchCallsTimes[i] + gpuTransferToCallsTimes[i] + gpuTransferFromCallsTimes[i];
+        gpuLaunchTransferSyncTimes[i] = gpuLaunchAndTransferCallsTimes[i];
+        if (isAsync) {
+            sdkStartTimer(&gpuSyncTimer);
+            {
+                if (hintsRequired) {
+                    *latch = 1;
+                }
+                checkCudaErrors(cudaStreamSynchronize(streamToRunOn));
+            }
+            sdkStopTimer(&gpuSyncTimer);
+            gpuLaunchTransferSyncTimes[i] += sdkGetAverageTimerValue(&gpuSyncTimer);
+            sdkResetTimer(&gpuSyncTimer);
+        }
+
+        sdkStartTimer(&cpuAccessTimer);
+        {
+            verifyMatrixData((i & 0x1 == 0) ? randValuesVerifyXmulY : randValuesVerifyYmulX, hptrC, matrixDim);
+        }
+        sdkStopTimer(&cpuAccessTimer);
+        cpuAccessTimes[i] += sdkGetAverageTimerValue(&cpuAccessTimer);
+        sdkResetTimer(&cpuAccessTimer);
+        overallTimes[i] = cpuAccessTimes[i] + gpuLaunchTransferSyncTimes[i];
+    }
+
+    switch (allocType) {
+    case USE_HOST_PAGEABLE_AND_DEVICE_MEMORY:
+    case USE_HOST_PAGEABLE_AND_DEVICE_MEMORY_ASYNC:
+        free(hptrA);
+        free(hptrB);
+        free(hptrC);
+        checkCudaErrors(cudaFree(dptrA));
+        checkCudaErrors(cudaFree(dptrB));
+        checkCudaErrors(cudaFree(dptrC));
+        break;
+
+    case USE_HOST_PAGELOCKED_AND_DEVICE_MEMORY:
+    case USE_HOST_PAGELOCKED_AND_DEVICE_MEMORY_ASYNC:
+        checkCudaErrors(cudaFreeHost(hptrA));
+        checkCudaErrors(cudaFreeHost(hptrB));
+        checkCudaErrors(cudaFreeHost(hptrC));
+        checkCudaErrors(cudaFree(dptrA));
+        checkCudaErrors(cudaFree(dptrB));
+        checkCudaErrors(cudaFree(dptrC));
+        break;
+
+    case USE_ZERO_COPY:
+        checkCudaErrors(cudaFreeHost(hptrA));
+        checkCudaErrors(cudaFreeHost(hptrB));
+        checkCudaErrors(cudaFreeHost(hptrC));
+        break;
+
+    case USE_MANAGED_MEMORY:
+    case USE_MANAGED_MEMORY_WITH_HINTS:
+    case USE_MANAGED_MEMORY_WITH_HINTS_ASYNC:
+        checkCudaErrors(cudaFree(dptrA));
+        checkCudaErrors(cudaFree(dptrB));
+        checkCudaErrors(cudaFree(dptrC));
+        break;
+
+    default:
+        exit(EXIT_FAILURE); // exit due to error
+    }
+
+    checkCudaErrors(cudaStreamDestroy(streamToRunOn));
+    checkCudaErrors(cudaFreeHost(latch));
+    free(randValuesX);
+    free(randValuesY);
+    free(randValuesVerifyXmulY);
+    free(randValuesVerifyYmulX);
+    sdkDeleteTimer(&gpuLaunchCallsTimer);
+    sdkDeleteTimer(&gpuTransferCallsTimer);
+    sdkDeleteTimer(&gpuSyncTimer);
+    sdkDeleteTimer(&cpuAccessTimer);
+}
+
+void matrixMultiplyPerfRunner(bool reportAsBandwidth,
+                              bool print_launch_transfer_results,
+                              bool print_std_deviation,
+                              int  device_id)
+{
+    int          i;
+    unsigned int minMatrixDim  = 32;
+    unsigned int multiplierDim = 2;
+    unsigned int matrixDim;
+    unsigned int minSize    = minMatrixDim * minMatrixDim * sizeof(float);
+    unsigned int maxSize    = (maxSampleSizeInMb * ONE_MB) / 4; // 3 buffers are used, but dividing by 4 (power of 2)
+    unsigned int multiplier = multiplierDim * multiplierDim;
+    unsigned int numSizesToTest;
+
+    struct testResults *results;
+    struct resultsData *gpuLaunchCallsTimes;
+    struct resultsData *gpuTransferToCallsTimes;
+    struct resultsData *gpuTransferFromCallsTimes;
+    struct resultsData *gpuLaunchAndTransferCallsTimes;
+    struct resultsData *gpuLaunchTransferSyncTimes;
+    struct resultsData *cpuAccessTimes;
+    struct resultsData *overallTimes;
+    unsigned long      *sizesToTest;
+    unsigned int        j;
+
+    numSizesToTest = findNumSizesToTest(minSize, maxSize, multiplier);
+
+    createAndInitTestResults(&results, "matrixMultiplyPerf", numKernelRuns, numSizesToTest);
+
+    sizesToTest = getPtrSizesToTest(results);
+
+    createResultDataAndAddToTestResults(
+        &gpuLaunchCallsTimes, results, "GPU Kernel Launch Call Time", false, reportAsBandwidth);
+    createResultDataAndAddToTestResults(
+        &gpuTransferToCallsTimes, results, "CPU to GPU Transfer Calls Time", false, reportAsBandwidth);
+    createResultDataAndAddToTestResults(
+        &gpuTransferFromCallsTimes, results, "GPU to CPU Transfer Calls Time", false, reportAsBandwidth);
+    createResultDataAndAddToTestResults(
+        &gpuLaunchAndTransferCallsTimes, results, "GPU Launch and Transfer Calls Time", false, reportAsBandwidth);
+    createResultDataAndAddToTestResults(
+        &gpuLaunchTransferSyncTimes, results, "GPU Launch Transfer and Sync Time", false, reportAsBandwidth);
+    createResultDataAndAddToTestResults(&cpuAccessTimes, results, "CPU Access Time", false, reportAsBandwidth);
+    createResultDataAndAddToTestResults(&overallTimes, results, "Overall Time", false, reportAsBandwidth);
+
+    printf("Running ");
+    for (matrixDim = minMatrixDim, j = 0; matrixDim * matrixDim <= maxSize / sizeof(float);
+         matrixDim *= multiplierDim, ++j) {
+        sizesToTest[j] = matrixDim * matrixDim * sizeof(float);
+        for (i = MEMALLOC_TYPE_START; i <= MEMALLOC_TYPE_END; i++) {
+            printf(".");
+            fflush(stdout);
+            runMatrixMultiplyKernel(matrixDim,
+                                    i,
+                                    numKernelRuns,
+                                    getPtrRunTimesInMs(gpuLaunchCallsTimes, i, j),
+                                    getPtrRunTimesInMs(gpuTransferToCallsTimes, i, j),
+                                    getPtrRunTimesInMs(gpuTransferFromCallsTimes, i, j),
+                                    getPtrRunTimesInMs(gpuLaunchAndTransferCallsTimes, i, j),
+                                    getPtrRunTimesInMs(gpuLaunchTransferSyncTimes, i, j),
+                                    getPtrRunTimesInMs(cpuAccessTimes, i, j),
+                                    getPtrRunTimesInMs(overallTimes, i, j),
+                                    device_id);
+        }
+    }
+    printf("\n");
+    printResults(results, print_launch_transfer_results, print_std_deviation);
+    freeTestResultsAndAllResultsData(results);
+}
+
+static void usage()
+{
+    printf("./cudaMemoryTypesPerf [-device=<device_id>] [-reportAsBandwidth] "
+           "[-print-launch-transfer-results] [-print-std-deviation] [-verbose]\n");
+    printf("Options:\n");
+    printf("-reportAsBandwidth:             By default time taken is printed, this "
+           "option allows to instead print bandwidth.\n");
+    printf("-print-launch-transfer-results: By default overall results are printed, "
+           "this option allows to print data transfers and kernel time as well.\n");
+    printf("-print-std-deviation:           Prints std deviation of the results.\n");
+    printf("-kernel-iterations=<num>:       Number of times the kernel tests should "
+           "be run[default is 100 iterations].\n");
+    printf("-device=<device_id>:            Allows to pass GPU Device ID on which "
+           "the tests will be run.\n");
+    printf("-verbose:                       Prints highly verbose output.\n");
+}
+
+int main(int argc, char **argv)
+{
+    bool reportAsBandwidth             = false;
+    bool print_launch_transfer_results = false;
+    bool print_std_deviation           = false;
+
+    if (checkCmdLineFlag(argc, (const char **)argv, "help") || checkCmdLineFlag(argc, (const char **)argv, "h")) {
+        usage();
+        printf("&&&& %s WAIVED\n", argv[0]);
+        exit(EXIT_WAIVED);
+    }
+
+    if (checkCmdLineFlag(argc, (const char **)argv, "reportAsBandwidth")) {
+        reportAsBandwidth = true;
+    }
+
+    if (checkCmdLineFlag(argc, (const char **)argv, "print-launch-transfer-results")) {
+        print_launch_transfer_results = true;
+    }
+
+    if (checkCmdLineFlag(argc, (const char **)argv, "print-std-deviation")) {
+        print_std_deviation = true;
+    }
+
+    if (checkCmdLineFlag(argc, (const char **)argv, "kernel-iterations")) {
+        numKernelRuns = getCmdLineArgumentInt(argc, (const char **)argv, "kernel-iterations");
+    }
+
+    if (checkCmdLineFlag(argc, (const char **)argv, "verbose")) {
+        verboseResults = 1;
+    }
+
+    // set device
+    cudaDeviceProp device_prop;
+    int            device_id = findCudaDevice(argc, (const char **)argv);
+    checkCudaErrors(cudaGetDeviceProperties(&device_prop, device_id));
+
+    if (!device_prop.managedMemory) {
+        // This samples requires being run on a device that supports Unified Memory
+        fprintf(stderr, "Unified Memory not supported on this device\n");
+
+        exit(EXIT_WAIVED);
+    }
+
+    matrixMultiplyPerfRunner(reportAsBandwidth, print_launch_transfer_results, print_std_deviation, device_id);
+
+    printf("\nNOTE: The CUDA Samples are not meant for performance measurements. "
+           "Results may vary when GPU Boost is enabled.\n");
+    exit(EXIT_SUCCESS);
+}

cpp/6_Performance/alignedTypes/.vscode/c_cpp_properties.json

@@ -0,0 +1,18 @@
+{
+    "configurations": [
+        {
+            "name": "Linux",
+            "includePath": [
+                "${workspaceFolder}/**",
+                "${workspaceFolder}/../../../Common"
+            ],
+            "defines": [],
+            "compilerPath": "/usr/local/cuda/bin/nvcc",
+            "cStandard": "gnu17",
+            "cppStandard": "gnu++14",
+            "intelliSenseMode": "linux-gcc-x64",
+            "configurationProvider": "ms-vscode.makefile-tools"
+        }
+    ],
+    "version": 4
+}

cpp/6_Performance/alignedTypes/.vscode/extensions.json

@@ -0,0 +1,7 @@
+{
+    "recommendations": [
+        "nvidia.nsight-vscode-edition",
+        "ms-vscode.cpptools",
+        "ms-vscode.makefile-tools"
+    ]
+}

cpp/6_Performance/alignedTypes/CMakeLists.txt

@@ -0,0 +1,34 @@
+cmake_minimum_required(VERSION 3.20)
+
+list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/../../../cmake/Modules")
+
+project(alignedTypes LANGUAGES C CXX CUDA)
+
+find_package(CUDAToolkit REQUIRED)
+
+set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+
+set(CMAKE_CUDA_ARCHITECTURES 75 80 86 87 89 90 100 110 120)
+set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Wno-deprecated-gpu-targets")
+if(ENABLE_CUDA_DEBUG)
+    set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -G")        # enable cuda-gdb (may significantly affect performance on some targets)
+else()
+    set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -lineinfo") # add line information to all builds for debug tools (exclusive to -G option)
+endif()
+
+# Include directories and libraries
+include_directories(../../../Common)
+
+# Source file
+# Add target for alignedTypes
+add_executable(alignedTypes alignedTypes.cu)
+
+target_compile_options(alignedTypes PRIVATE $<$<COMPILE_LANGUAGE:CUDA>:--extended-lambda>)
+
+target_compile_features(alignedTypes PRIVATE cxx_std_17 cuda_std_17)
+
+set_target_properties(alignedTypes PROPERTIES CUDA_SEPARABLE_COMPILATION ON)
+
+# Include installation configuration
+include(${CMAKE_CURRENT_SOURCE_DIR}/../../../cmake/InstallSamples.cmake)
+setup_samples_install()

cpp/6_Performance/alignedTypes/README.md

@@ -0,0 +1,32 @@
+# alignedTypes - Aligned Types
+
+## Description
+
+A simple test, showing huge access speed gap between aligned and misaligned structures. It measures per-element copy throughput for aligned and misaligned structures on big chunks of data.
+
+## Key Concepts
+
+Performance Strategies
+
+## Supported SM Architectures
+
+[SM 5.0 ](https://developer.nvidia.com/cuda-gpus)  [SM 5.2 ](https://developer.nvidia.com/cuda-gpus)  [SM 5.3 ](https://developer.nvidia.com/cuda-gpus)  [SM 6.0 ](https://developer.nvidia.com/cuda-gpus)  [SM 6.1 ](https://developer.nvidia.com/cuda-gpus)  [SM 7.0 ](https://developer.nvidia.com/cuda-gpus)  [SM 7.2 ](https://developer.nvidia.com/cuda-gpus)  [SM 7.5 ](https://developer.nvidia.com/cuda-gpus)  [SM 8.0 ](https://developer.nvidia.com/cuda-gpus)  [SM 8.6 ](https://developer.nvidia.com/cuda-gpus)  [SM 8.7 ](https://developer.nvidia.com/cuda-gpus)  [SM 8.9 ](https://developer.nvidia.com/cuda-gpus)  [SM 9.0 ](https://developer.nvidia.com/cuda-gpus)
+
+## Supported OSes
+
+Linux, Windows
+
+## Supported CPU Architecture
+
+x86_64, armv7l
+
+## CUDA APIs involved
+
+### [CUDA Runtime API](http://docs.nvidia.com/cuda/cuda-runtime-api/index.html)
+cudaMemcpy, cudaFree, cudaDeviceSynchronize, cudaMemset, cudaMalloc, cudaGetDeviceProperties
+
+## Prerequisites
+
+Download and install the [CUDA Toolkit](https://developer.nvidia.com/cuda-downloads) for your corresponding platform.
+
+## References (for more details)

cpp/6_Performance/alignedTypes/alignedTypes.cu

@@ -0,0 +1,313 @@
+/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * This is a simple test showing huge access speed gap
+ * between aligned and misaligned structures
+ * (those having/missing __align__ keyword).
+ * It measures per-element copy throughput for
+ * aligned and misaligned structures on
+ * big chunks of data.
+ */
+
+// includes, system
+#include <math.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+// includes, project
+#include <helper_cuda.h>      // helper functions for CUDA error checking and initialization
+#include <helper_functions.h> // helper utility functions
+
+////////////////////////////////////////////////////////////////////////////////
+// Misaligned types
+////////////////////////////////////////////////////////////////////////////////
+typedef unsigned char uint8;
+
+typedef unsigned short int uint16;
+
+typedef struct
+{
+    unsigned char r, g, b, a;
+} RGBA8_misaligned;
+
+typedef struct
+{
+    unsigned int l, a;
+} LA32_misaligned;
+
+typedef struct
+{
+    unsigned int r, g, b;
+} RGB32_misaligned;
+
+typedef struct
+{
+    unsigned int r, g, b, a;
+} RGBA32_misaligned;
+
+////////////////////////////////////////////////////////////////////////////////
+// Aligned types
+////////////////////////////////////////////////////////////////////////////////
+typedef struct __align__(4)
+{
+    unsigned char r, g, b, a;
+} RGBA8;
+
+typedef unsigned int I32;
+
+typedef struct __align__(8)
+{
+    unsigned int l, a;
+} LA32;
+
+typedef struct __align__(16)
+{
+    unsigned int r, g, b;
+} RGB32;
+
+typedef struct __align__(16)
+{
+    unsigned int r, g, b, a;
+} RGBA32;
+
+////////////////////////////////////////////////////////////////////////////////
+// Because G80 class hardware natively supports global memory operations
+// only with data elements of 4, 8 and 16 bytes, if structure size
+// exceeds 16 bytes, it can't be efficiently read or written,
+// since more than one global memory non-coalescable load/store instructions
+// will be generated, even if __align__ option is supplied.
+// "Structure of arrays" storage strategy offers best performance
+// in general case. See section 5.1.2 of the Programming Guide.
+////////////////////////////////////////////////////////////////////////////////
+typedef struct __align__(16)
+{
+    RGBA32 c1, c2;
+} RGBA32_2;
+
+////////////////////////////////////////////////////////////////////////////////
+// Common host and device functions
+////////////////////////////////////////////////////////////////////////////////
+// Round a / b to nearest higher integer value
+int iDivUp(int a, int b) { return (a % b != 0) ? (a / b + 1) : (a / b); }
+
+// Round a / b to nearest lower integer value
+int iDivDown(int a, int b) { return a / b; }
+
+// Align a to nearest higher multiple of b
+int iAlignUp(int a, int b) { return (a % b != 0) ? (a - a % b + b) : a; }
+
+// Align a to nearest lower multiple of b
+int iAlignDown(int a, int b) { return a - a % b; }
+
+////////////////////////////////////////////////////////////////////////////////
+// Simple CUDA kernel.
+// Copy is carried out on per-element basis,
+// so it's not per-byte in case of padded structures.
+////////////////////////////////////////////////////////////////////////////////
+template <class TData> __global__ void testKernel(TData *d_odata, TData *d_idata, int numElements)
+{
+    const int tid        = blockDim.x * blockIdx.x + threadIdx.x;
+    const int numThreads = blockDim.x * gridDim.x;
+
+    for (int pos = tid; pos < numElements; pos += numThreads) {
+        d_odata[pos] = d_idata[pos];
+    }
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// Validation routine for simple copy kernel.
+// We must know "packed" size of TData (number_of_fields * sizeof(simple_type))
+// and compare only these "packed" parts of the structure,
+// containing actual user data. The compiler behavior with padding bytes
+// is undefined, since padding is merely a placeholder
+// and doesn't contain any user data.
+////////////////////////////////////////////////////////////////////////////////
+template <class TData> int testCPU(TData *h_odata, TData *h_idata, int numElements, int packedElementSize)
+{
+    for (int pos = 0; pos < numElements; pos++) {
+        TData src = h_idata[pos];
+        TData dst = h_odata[pos];
+
+        for (int i = 0; i < packedElementSize; i++)
+            if (((char *)&src)[i] != ((char *)&dst)[i]) {
+                return 0;
+            }
+    }
+
+    return 1;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// Data configuration
+////////////////////////////////////////////////////////////////////////////////
+// Memory chunk size in bytes. Reused for test
+const int MEM_SIZE       = 50000000;
+const int NUM_ITERATIONS = 32;
+
+// GPU input and output data
+unsigned char *d_idata, *d_odata;
+// CPU input data and instance of GPU output data
+unsigned char      *h_idataCPU, *h_odataGPU;
+StopWatchInterface *hTimer = NULL;
+
+template <class TData> int runTest(int packedElementSize, int memory_size)
+{
+    const int totalMemSizeAligned = iAlignDown(memory_size, sizeof(TData));
+    const int numElements         = iDivDown(memory_size, sizeof(TData));
+
+    // Clean output buffer before current test
+    checkCudaErrors(cudaMemset(d_odata, 0, memory_size));
+    // Run test
+    checkCudaErrors(cudaDeviceSynchronize());
+    sdkResetTimer(&hTimer);
+    sdkStartTimer(&hTimer);
+
+    for (int i = 0; i < NUM_ITERATIONS; i++) {
+        testKernel<TData><<<64, 256>>>((TData *)d_odata, (TData *)d_idata, numElements);
+        getLastCudaError("testKernel() execution failed\n");
+    }
+
+    checkCudaErrors(cudaDeviceSynchronize());
+    sdkStopTimer(&hTimer);
+    double gpuTime = sdkGetTimerValue(&hTimer) / NUM_ITERATIONS;
+    printf("Avg. time: %f ms / Copy throughput: %f GB/s.\n",
+           gpuTime,
+           (double)totalMemSizeAligned / (gpuTime * 0.001 * 1073741824.0));
+
+    // Read back GPU results and run validation
+    checkCudaErrors(cudaMemcpy(h_odataGPU, d_odata, memory_size, cudaMemcpyDeviceToHost));
+    int flag = testCPU((TData *)h_odataGPU, (TData *)h_idataCPU, numElements, packedElementSize);
+
+    printf(flag ? "\tTEST OK\n" : "\tTEST FAILURE\n");
+
+    return !flag;
+}
+
+int main(int argc, char **argv)
+{
+    int i, nTotalFailures = 0;
+
+    int            devID;
+    cudaDeviceProp deviceProp;
+    printf("[%s] - Starting...\n", argv[0]);
+
+    // find first CUDA device
+    devID = findCudaDevice(argc, (const char **)argv);
+
+    // get number of SMs on this GPU
+    checkCudaErrors(cudaGetDeviceProperties(&deviceProp, devID));
+    printf("[%s] has %d MP(s) x %d (Cores/MP) = %d (Cores)\n",
+           deviceProp.name,
+           deviceProp.multiProcessorCount,
+           _ConvertSMVer2Cores(deviceProp.major, deviceProp.minor),
+           _ConvertSMVer2Cores(deviceProp.major, deviceProp.minor) * deviceProp.multiProcessorCount);
+
+    // Anything that is less than 192 Cores will have a scaled down workload
+    float scale_factor = max(
+        (192.0f / (_ConvertSMVer2Cores(deviceProp.major, deviceProp.minor) * (float)deviceProp.multiProcessorCount)),
+        1.0f);
+
+    int MemorySize = (int)(MEM_SIZE / scale_factor) & 0xffffff00; // force multiple of 256 bytes
+
+    printf("> Compute scaling value = %4.2f\n", scale_factor);
+    printf("> Memory Size = %d\n", MemorySize);
+
+    sdkCreateTimer(&hTimer);
+
+    printf("Allocating memory...\n");
+    h_idataCPU = (unsigned char *)malloc(MemorySize);
+    h_odataGPU = (unsigned char *)malloc(MemorySize);
+    checkCudaErrors(cudaMalloc((void **)&d_idata, MemorySize));
+    checkCudaErrors(cudaMalloc((void **)&d_odata, MemorySize));
+
+    printf("Generating host input data array...\n");
+
+    for (i = 0; i < MemorySize; i++) {
+        h_idataCPU[i] = (i & 0xFF) + 1;
+    }
+
+    printf("Uploading input data to GPU memory...\n");
+    checkCudaErrors(cudaMemcpy(d_idata, h_idataCPU, MemorySize, cudaMemcpyHostToDevice));
+
+    printf("Testing misaligned types...\n");
+    printf("uint8...\n");
+    nTotalFailures += runTest<uint8>(1, MemorySize);
+
+    printf("uint16...\n");
+    nTotalFailures += runTest<uint16>(2, MemorySize);
+
+    printf("RGBA8_misaligned...\n");
+    nTotalFailures += runTest<RGBA8_misaligned>(4, MemorySize);
+
+    printf("LA32_misaligned...\n");
+    nTotalFailures += runTest<LA32_misaligned>(8, MemorySize);
+
+    printf("RGB32_misaligned...\n");
+    nTotalFailures += runTest<RGB32_misaligned>(12, MemorySize);
+
+    printf("RGBA32_misaligned...\n");
+    nTotalFailures += runTest<RGBA32_misaligned>(16, MemorySize);
+
+    printf("Testing aligned types...\n");
+    printf("RGBA8...\n");
+    nTotalFailures += runTest<RGBA8>(4, MemorySize);
+
+    printf("I32...\n");
+    nTotalFailures += runTest<I32>(4, MemorySize);
+
+    printf("LA32...\n");
+    nTotalFailures += runTest<LA32>(8, MemorySize);
+
+    printf("RGB32...\n");
+    nTotalFailures += runTest<RGB32>(12, MemorySize);
+
+    printf("RGBA32...\n");
+    nTotalFailures += runTest<RGBA32>(16, MemorySize);
+
+    printf("RGBA32_2...\n");
+    nTotalFailures += runTest<RGBA32_2>(32, MemorySize);
+
+    printf("\n[alignedTypes] -> Test Results: %d Failures\n", nTotalFailures);
+
+    printf("Shutting down...\n");
+    checkCudaErrors(cudaFree(d_idata));
+    checkCudaErrors(cudaFree(d_odata));
+    free(h_odataGPU);
+    free(h_idataCPU);
+
+    sdkDeleteTimer(&hTimer);
+
+    if (nTotalFailures != 0) {
+        printf("Test failed!\n");
+        exit(EXIT_FAILURE);
+    }
+
+    printf("Test passed\n");
+    exit(EXIT_SUCCESS);
+}

cpp/6_Performance/alignedTypes/doc/alignedTypes.txt

@@ -0,0 +1,10 @@
+CUDA programming language, being a C with extensions, offers the ability to use arbitrary data structures in GPU programs. But in order for the hardware to perform efficient global loads and stores with variables of structured types, additional alignment details must be specified.
+
+Take a look at this structure definition:
+typedef struct{
+    float a;
+    float b;
+} testStructure;
+
+Without alignment specification the compiler will not automatically use a single 64-bit global memory load/store instruction, but will emit two 32-bit load instructions instead.
+This significantly impacts aggregate load/store bandwidth, since the latter breaks coalescing rules because of incontiguous memory access pattern. Refer to section 5.1.2.1 of the Programming Guide.

cpp/6_Performance/cudaGraphsPerfScaling/.vscode/c_cpp_properties.json

@@ -0,0 +1,18 @@
+{
+    "configurations": [
+        {
+            "name": "Linux",
+            "includePath": [
+                "${workspaceFolder}/**",
+                "${workspaceFolder}/../../../Common"
+            ],
+            "defines": [],
+            "compilerPath": "/usr/local/cuda/bin/nvcc",
+            "cStandard": "gnu17",
+            "cppStandard": "gnu++14",
+            "intelliSenseMode": "linux-gcc-x64",
+            "configurationProvider": "ms-vscode.makefile-tools"
+        }
+    ],
+    "version": 4
+}

cpp/6_Performance/cudaGraphsPerfScaling/.vscode/extensions.json

@@ -0,0 +1,7 @@
+{
+    "recommendations": [
+        "nvidia.nsight-vscode-edition",
+        "ms-vscode.cpptools",
+        "ms-vscode.makefile-tools"
+    ]
+}

cpp/6_Performance/cudaGraphsPerfScaling/CMakeLists.txt

@@ -0,0 +1,34 @@
+cmake_minimum_required(VERSION 3.20)
+
+list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/../../../cmake/Modules")
+
+project(cudaGraphsPerfScaling LANGUAGES C CXX CUDA)
+
+find_package(CUDAToolkit REQUIRED)
+
+set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+
+set(CMAKE_CUDA_ARCHITECTURES 75 80 86 87 89 90 100 110 120)
+set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Wno-deprecated-gpu-targets")
+if(ENABLE_CUDA_DEBUG)
+    set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -G")        # enable cuda-gdb (may significantly affect performance on some targets)
+else()
+    set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -lineinfo") # add line information to all builds for debug tools (exclusive to -G option)
+endif()
+
+# Include directories and libraries
+include_directories(../../../Common)
+
+# Source file
+# Add target for cudaGraphsPerfScaling
+add_executable(cudaGraphsPerfScaling cudaGraphPerfScaling.cu)
+
+target_compile_options(cudaGraphsPerfScaling PRIVATE $<$<COMPILE_LANGUAGE:CUDA>:--extended-lambda>)
+
+target_compile_features(cudaGraphsPerfScaling PRIVATE cxx_std_17 cuda_std_17)
+
+set_target_properties(cudaGraphsPerfScaling PROPERTIES CUDA_SEPARABLE_COMPILATION ON)
+
+# Include installation configuration
+include(${CMAKE_CURRENT_SOURCE_DIR}/../../../cmake/InstallSamples.cmake)
+setup_samples_install()

cpp/6_Performance/cudaGraphsPerfScaling/README.md

@@ -0,0 +1,32 @@
+# cudaGraphsPerfScaling - Cuda Graphs Perf Scaling
+
+## Description
+
+A simple program for characterizing cuda graph api performance with different sized graphs.
+
+## Key Concepts
+
+Performance Strategies
+
+## Supported SM Architectures
+
+[SM 5.0 ](https://developer.nvidia.com/cuda-gpus)  [SM 5.2 ](https://developer.nvidia.com/cuda-gpus)  [SM 5.3 ](https://developer.nvidia.com/cuda-gpus)  [SM 6.0 ](https://developer.nvidia.com/cuda-gpus)  [SM 6.1 ](https://developer.nvidia.com/cuda-gpus)  [SM 7.0 ](https://developer.nvidia.com/cuda-gpus)  [SM 7.2 ](https://developer.nvidia.com/cuda-gpus)  [SM 7.5 ](https://developer.nvidia.com/cuda-gpus)  [SM 8.0 ](https://developer.nvidia.com/cuda-gpus)  [SM 8.6 ](https://developer.nvidia.com/cuda-gpus)  [SM 8.7 ](https://developer.nvidia.com/cuda-gpus)  [SM 8.9 ](https://developer.nvidia.com/cuda-gpus)  [SM 9.0 ](https://developer.nvidia.com/cuda-gpus)
+
+## Supported OSes
+
+Linux, Windows
+
+## Supported CPU Architecture
+
+x86_64, armv7l
+
+## CUDA APIs involved
+
+### [CUDA Runtime API](http://docs.nvidia.com/cuda/cuda-runtime-api/index.html)
+cudaStreamBeginCapture, cudaGraphInstantiate, cudaGraphLaunch, cudaGraphUpload
+
+## Prerequisites
+
+Download and install the [CUDA Toolkit](https://developer.nvidia.com/cuda-downloads) for your corresponding platform.
+
+## References (for more details)

cpp/6_Performance/cudaGraphsPerfScaling/cudaGraphPerfScaling.cu

@@ -0,0 +1,439 @@
+/* Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * This is a simple application showing the performance characteristics of cudaGraphs.
+ */
+
+#define USE_NVTX
+
+#include <chrono>
+#include <cstdio>
+#include <cuda_runtime.h>
+#include <vector>
+
+typedef volatile int LatchType;
+
+std::chrono::time_point<std::chrono::high_resolution_clock> getCpuTime()
+{
+    return std::chrono::high_resolution_clock::now();
+}
+
+template <typename T> float getMicroSecondDuration(T start, T end)
+{
+    return std::chrono::duration_cast<std::chrono::nanoseconds>(end - start).count() * .001f;
+}
+
+float getAsyncMicroSecondDuration(cudaEvent_t start, cudaEvent_t end)
+{
+    float ms;
+    cudaEventElapsedTime(&ms, start, end);
+    return ms * 1000;
+}
+
+#ifdef USE_NVTX
+#include <nvtx3/nvToolsExt.h>
+
+class Tracer
+{
+public:
+    Tracer(const char *name) { nvtxRangePushA(name); }
+    ~Tracer() { nvtxRangePop(); }
+};
+#define RANGE(name)      Tracer uniq_name_using_macros(name);
+#define RANGE_PUSH(name) nvtxRangePushA(name)
+#define RANGE_POP()      nvtxRangePop();
+#else
+#define RANGE(name)
+#endif
+
+std::vector<cudaStream_t> stream;
+cudaEvent_t               event[1];
+cudaEvent_t               timingEvent[2];
+
+struct hostData
+{
+    long long timeElapsed;
+    bool      timeoutDetected;
+    long long timeElapsed2;
+    bool      timeoutDetected2;
+    LatchType latch;
+    LatchType latch2;
+};
+
+struct hostData *hostData;
+
+__global__ void empty() {}
+
+// Function to read the GPU nanosecond timer in a kernel
+__device__ __forceinline__ unsigned long long __globaltimer()
+{
+    unsigned long long globaltimer;
+    asm volatile("mov.u64 %0, %globaltimer;" : "=l"(globaltimer));
+    return globaltimer;
+}
+
+__global__ void delay(long long ticks)
+{
+    long long endTime = clock64() + ticks;
+    while (clock64() < endTime)
+        ;
+}
+
+__global__ void waitWithTimeout(long long nanoseconds, bool *timeoutDetected, long long *timeElapsed, LatchType *latch)
+{
+    long long startTime = __globaltimer();
+    long long endTime   = startTime + nanoseconds;
+    long long time      = 0;
+    do {
+        time = __globaltimer();
+    } while (time < endTime && (latch == NULL || *latch == 0));
+    if (timeElapsed != NULL) {
+        *timeElapsed = time - startTime;
+    }
+    if (timeoutDetected) {
+        // report timeout if latch not detected
+        *timeoutDetected = (latch == NULL || *latch == 0);
+    }
+}
+
+__global__ void preUploadAnnotation() {}
+
+__global__ void postUploadAnnotation() {}
+
+cudaGraph_t createParallelChain(int length, int width, bool singleEntry = false)
+{
+    RANGE_PUSH(__func__);
+    RANGE("capture");
+    cudaGraph_t graph;
+    cudaStreamBeginCapture(stream[0], cudaStreamCaptureModeGlobal);
+    int streamIdx = 0;
+    if (singleEntry) {
+        empty<<<1, 1, 0, stream[streamIdx]>>>();
+    }
+
+    cudaEventRecord(event[0], stream[0]);
+    for (int i = 1; i < width; i++) {
+        cudaStreamWaitEvent(stream[i], event[0]);
+    }
+
+    for (int i = 0; i < width; i++) {
+        streamIdx = i;
+        for (int j = 0; j < length; j++) {
+            empty<<<1, 1, 0, stream[streamIdx]>>>();
+        }
+    }
+
+    for (int i = 1; i < width; i++) {
+        cudaEventRecord(event[0], stream[i]);
+        cudaStreamWaitEvent(stream[0], event[0]);
+    }
+
+    cudaStreamEndCapture(stream[0], &graph);
+    return graph;
+}
+
+std::vector<const char *> metricName;
+std::vector<float>        metricValue;
+
+int  counter2 = 0;
+void runDemo(cudaGraph_t graph, int length, int width)
+{
+    cudaGraphExec_t graphExec;
+    {
+        auto start = getCpuTime();
+        cudaGraphInstantiateWithFlags(&graphExec, graph, 0);
+        auto end = getCpuTime();
+        metricName.push_back("instantiation");
+        metricValue.push_back(getMicroSecondDuration(start, end));
+    }
+    {
+        RANGE("launch including upload");
+        auto start = getCpuTime();
+        cudaGraphLaunch(graphExec, stream[0]);
+        auto apiReturn = getCpuTime();
+        cudaStreamSynchronize(stream[0]);
+        auto streamSync = getCpuTime();
+        metricName.push_back("first_launch_api");
+        metricValue.push_back(getMicroSecondDuration(start, apiReturn));
+        metricName.push_back("first_launch_total");
+        metricValue.push_back(getMicroSecondDuration(start, streamSync));
+    }
+    {
+        RANGE("repeat lauch in empty stream");
+        auto start = getCpuTime();
+        cudaGraphLaunch(graphExec, stream[0]);
+        auto apiReturn = getCpuTime();
+        cudaStreamSynchronize(stream[0]);
+        auto streamSync = getCpuTime();
+        metricName.push_back("repeat_launch_api");
+        metricValue.push_back(getMicroSecondDuration(start, apiReturn));
+        metricName.push_back("repeat_launch_total");
+        metricValue.push_back(getMicroSecondDuration(start, streamSync));
+    }
+    {
+        // re-instantiating the exec to simulate first launch into a busy stream.
+        cudaGraphExecDestroy(graphExec);
+        cudaGraphInstantiateWithFlags(&graphExec, graph, 0);
+
+        long long maxTimeoutNanoSeconds = 4000 + 500 * length * width;
+        waitWithTimeout<<<1, 1, 0, stream[0]>>>(
+            maxTimeoutNanoSeconds, &hostData->timeoutDetected, &hostData->timeElapsed, &hostData->latch);
+
+        RANGE("launch including upload in busy stream");
+        cudaEventRecord(timingEvent[0], stream[0]);
+        cudaGraphLaunch(graphExec, stream[0]);
+        cudaEventRecord(timingEvent[1], stream[0]);
+
+        hostData->latch = 1;
+        cudaStreamSynchronize(stream[0]);
+
+        metricName.push_back("first_launch_device");
+        metricValue.push_back(getAsyncMicroSecondDuration(timingEvent[0], timingEvent[1]));
+        metricName.push_back("blockingKernelTimeoutDetected");
+        metricValue.push_back(hostData->timeoutDetected);
+        hostData->latch           = 0;
+        hostData->timeoutDetected = 0;
+    }
+    {
+        RANGE("repeat lauch in busy stream");
+        long long maxTimeoutNanoSeconds = 4000 + 500 * length * width;
+        waitWithTimeout<<<1, 1, 0, stream[0]>>>(
+            maxTimeoutNanoSeconds, &hostData->timeoutDetected, &hostData->timeElapsed, &hostData->latch);
+        cudaEventRecord(timingEvent[0], stream[0]);
+        cudaGraphLaunch(graphExec, stream[0]);
+        cudaEventRecord(timingEvent[1], stream[0]);
+
+        hostData->latch = 1;
+        cudaStreamSynchronize(stream[0]);
+
+        metricName.push_back("repeat_launch_device");
+        metricValue.push_back(getAsyncMicroSecondDuration(timingEvent[0], timingEvent[1]));
+        metricName.push_back("blockingKernelTimeoutDetected");
+        metricValue.push_back(hostData->timeoutDetected);
+        hostData->latch           = 0;
+        hostData->timeoutDetected = 0;
+    }
+    {
+        // re-instantiating the exec to provide upload with work to do.
+        cudaGraphExecDestroy(graphExec);
+        cudaGraphInstantiateWithFlags(&graphExec, graph, 0);
+        long long maxTimeoutNanoSeconds = 4000 + 1000 * length * width;
+        waitWithTimeout<<<1, 1, 0, stream[0]>>>(
+            maxTimeoutNanoSeconds, &hostData->timeoutDetected2, &hostData->timeElapsed2, &hostData->latch2);
+        maxTimeoutNanoSeconds = 2000 + 500 * length * width;
+        waitWithTimeout<<<1, 1, 0, stream[1]>>>(
+            maxTimeoutNanoSeconds, &hostData->timeoutDetected, &hostData->timeElapsed, &hostData->latch);
+
+        RANGE("uploading a graph off of the critical path");
+        preUploadAnnotation<<<1, 1, 0, stream[1]>>>();
+        cudaEventRecord(timingEvent[0], stream[0]);
+        auto start = getCpuTime();
+        cudaGraphUpload(graphExec, stream[1]);
+        auto apiReturn = getCpuTime();
+        cudaEventRecord(event[0], stream[1]);
+        cudaEventRecord(timingEvent[1], stream[0]);
+        postUploadAnnotation<<<1, 1, 0, stream[1]>>>();
+
+        hostData->latch = 1; // release the blocking kernel for the upload
+        cudaStreamWaitEvent(stream[0], event[0]);
+        cudaGraphLaunch(graphExec, stream[0]);
+        cudaEventSynchronize(event[0]); // upload done, similuate critical path being ready for the graph to run by the
+                                        // release of the second latch
+
+        hostData->latch2 = 1; // release the work
+        cudaStreamSynchronize(stream[0]);
+
+        metricName.push_back("upload_api_time");
+        metricValue.push_back(getMicroSecondDuration(start, apiReturn));
+        metricName.push_back("updoad_device_time");
+        metricValue.push_back(getAsyncMicroSecondDuration(timingEvent[0], timingEvent[1]));
+        metricName.push_back("blockingKernelTimeoutDetected");
+        metricValue.push_back(hostData->timeoutDetected);
+
+        hostData->latch            = 0;
+        hostData->latch2           = 0;
+        hostData->timeoutDetected  = 0;
+        hostData->timeoutDetected2 = 0;
+    }
+    cudaGraphExecDestroy(graphExec);
+    cudaGraphDestroy(graph);
+    RANGE_POP();
+}
+
+void usage()
+{
+    printf("programName [outputFmt] [numTrials] [length] [width] [pattern] [stride] [maxLength] \n");
+    printf("\toutputFmt - program output, default=3 (see below)\n");
+    printf("\tnumTrials (per length)\n");
+    printf("\tstarting length of the topology\n");
+    printf("\twidth - width of the graph topology\n");
+    printf("\tpattern - Structure of graph, default=0 (see below)\n");
+    printf("\tstride - how to grow the length between each set of trials \n");
+    printf("\tmaxLength - maximum lenght to try \n");
+    printf("\n");
+    printf("outputFmt can be:\n");
+    printf("\t0: this help message\n");
+    printf("\t1: csv data headers\n");
+    printf("\t2: per trial csv data\n");
+    printf("\t3: csv data & headers\n");
+    printf("\t4: csv data is printed and trials are averaged for each length\n");
+    printf("\t5: csv data is printed and trials are averaged for each length and headers are printed\n");
+    printf("\n");
+    printf("Pattern can be:\n");
+    printf("\t0: No interconnect between branches\n");
+    printf("\t1: Adds an extra root node before the initial fork\n");
+}
+
+int main(int argc, char **argv)
+{
+    if (argc < 1) {
+        usage();
+        return 0;
+    }
+
+    int numTrials = 1, length = 20, width = 1, outputFmt = 3, pattern = 0, stride = 1;
+    if (argc > 1)
+        outputFmt = atoi(argv[1]);
+    if (argc > 2)
+        numTrials = atoi(argv[2]);
+    if (argc > 3)
+        length = atoi(argv[3]);
+    if (argc > 4)
+        width = atoi(argv[4]);
+    if (argc > 5)
+        pattern = atoi(argv[5]);
+    if (argc > 6)
+        stride = atoi(argv[6]);
+    int maxLength = length;
+    if (argc > 7)
+        maxLength = atoi(argv[7]);
+    if (maxLength < length) {
+        maxLength = length;
+    }
+
+    if ((outputFmt & 4) && (outputFmt & 2)) {
+        printf("printing average and all samples doesn't make sense\n");
+    }
+
+    if (length == 0 || width == 0 || outputFmt == 0 || outputFmt > 5 || pattern > 1) {
+        usage();
+        return 0;
+    }
+
+    bool singleEntry = (pattern == 1);
+
+    cudaGraph_t graph;
+
+    cudaFree(0);
+    cudaMallocHost(&hostData, sizeof(*hostData));
+    int numStreams = width;
+    if (numStreams == 1)
+        numStreams = 2; // demo needs two streams even if capture only needs 1.
+    stream.resize(numStreams);
+    for (int i = 0; i < numStreams; i++) {
+        cudaStreamCreate(&stream[i]);
+    }
+
+    cudaEventCreate(&event[0], cudaEventDisableTiming);
+    cudaEventCreate(&timingEvent[0], 0);
+    cudaEventCreate(&timingEvent[1], 0);
+
+    {
+        RANGE("warmup");
+        for (int i = 0; i < width; i++) {
+            empty<<<1, 1, 0, stream[i]>>>();
+        }
+        cudaStreamSynchronize(stream[0]);
+
+        auto start = getCpuTime();
+        graph      = createParallelChain(length, width, singleEntry);
+        auto end   = getCpuTime();
+        metricValue.push_back(getMicroSecondDuration(start, end));
+        metricName.push_back("capture");
+        runDemo(graph, length, width);
+    }
+
+    if (outputFmt & 1) {
+        printf("length, width, pattern, ");
+        for (int i = 0; i < metricName.size(); i++) {
+            printf("%s, ", metricName[i]);
+        }
+        printf("\r\n");
+    }
+
+    if (!(outputFmt & 6)) {
+        printf("skipping trials since no output is expected\n");
+        return 1;
+    }
+
+    std::vector<double> metricTotal;
+    metricTotal.resize(metricValue.size());
+
+    while (length <= maxLength) {
+        for (int i = 0; i < numTrials; i++) {
+            metricName.clear();
+            metricValue.clear();
+            auto start = getCpuTime();
+            graph      = createParallelChain(length, width, singleEntry);
+            auto end   = getCpuTime();
+            metricValue.push_back(getMicroSecondDuration(start, end));
+
+            runDemo(graph, length, width);
+
+            if (outputFmt & 2) {
+                printf("%d, %d, %d, ", length, width, pattern);
+                for (int i = 0; i < metricValue.size(); i++) {
+                    printf("%0.3f, ", metricValue[i]);
+                }
+                printf("\r\n");
+            }
+            if (outputFmt & 4) {
+                for (int i = 0; i < metricTotal.size(); i++) {
+                    metricTotal[i] += metricValue[i];
+                }
+            }
+        }
+
+        if (outputFmt & 4) {
+            printf("%d, %d, %d, ", length, width, pattern);
+            for (int i = 0; i < metricTotal.size(); i++) {
+                printf("%0.3f, ", metricTotal[i] / numTrials);
+                metricTotal[i] = 0;
+            }
+            printf("\r\n");
+        }
+
+        length += stride;
+    }
+
+    cudaFreeHost(hostData);
+
+    printf("\n");
+    printf("Test passed\n");
+    return 0;
+}

cpp/6_Performance/cudaGraphsPerfScaling/dataCollection.bash

@@ -0,0 +1,17 @@
+GPU=$1
+DRIVER_VERSION=$2
+BINARY=./cudaGraphsPerfScaling
+datadir=PERF_DATA
+
+suffix=$DRIVER_VERSION
+prefix=$GPU
+mkdir -p $datadir
+
+trials=600
+
+width=1
+nvidia-smi > $datadir/${prefix}_info_${suffix}.txt
+$BINARY 5 $trials 1 $width 0 1 256 > $datadir/${prefix}_${width}_small_${suffix}.csv
+$BINARY 5 $trials 1 $width 0 32 2048 > $datadir/${prefix}_${width}_large_${suffix}.csv
+width=4
+$BINARY 5 $trials 1 $width 0 1 256 > $datadir/${prefix}_${width}_small_${suffix}.csv

cpp/6_Performance/transpose/.vscode/c_cpp_properties.json

@@ -0,0 +1,18 @@
+{
+    "configurations": [
+        {
+            "name": "Linux",
+            "includePath": [
+                "${workspaceFolder}/**",
+                "${workspaceFolder}/../../../Common"
+            ],
+            "defines": [],
+            "compilerPath": "/usr/local/cuda/bin/nvcc",
+            "cStandard": "gnu17",
+            "cppStandard": "gnu++14",
+            "intelliSenseMode": "linux-gcc-x64",
+            "configurationProvider": "ms-vscode.makefile-tools"
+        }
+    ],
+    "version": 4
+}

cpp/6_Performance/transpose/.vscode/extensions.json

@@ -0,0 +1,7 @@
+{
+    "recommendations": [
+        "nvidia.nsight-vscode-edition",
+        "ms-vscode.cpptools",
+        "ms-vscode.makefile-tools"
+    ]
+}

cpp/6_Performance/transpose/CMakeLists.txt

@@ -0,0 +1,34 @@
+cmake_minimum_required(VERSION 3.20)
+
+list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/../../../cmake/Modules")
+
+project(transpose LANGUAGES C CXX CUDA)
+
+find_package(CUDAToolkit REQUIRED)
+
+set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+
+set(CMAKE_CUDA_ARCHITECTURES 75 80 86 87 89 90 100 110 120)
+set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Wno-deprecated-gpu-targets")
+if(ENABLE_CUDA_DEBUG)
+    set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -G")        # enable cuda-gdb (may significantly affect performance on some targets)
+else()
+    set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -lineinfo") # add line information to all builds for debug tools (exclusive to -G option)
+endif()
+
+# Include directories and libraries
+include_directories(../../../Common)
+
+# Source file
+# Add target for transpose
+add_executable(transpose transpose.cu)
+
+target_compile_options(transpose PRIVATE $<$<COMPILE_LANGUAGE:CUDA>:--extended-lambda>)
+
+target_compile_features(transpose PRIVATE cxx_std_17 cuda_std_17)
+
+set_target_properties(transpose PROPERTIES CUDA_SEPARABLE_COMPILATION ON)
+
+# Include installation configuration
+include(${CMAKE_CURRENT_SOURCE_DIR}/../../../cmake/InstallSamples.cmake)
+setup_samples_install()

cpp/6_Performance/transpose/README.md

@@ -0,0 +1,34 @@
+# transpose - Matrix Transpose
+
+## Description
+
+This sample demonstrates Matrix Transpose.  Different performance are shown to achieve high performance.
+
+## Key Concepts
+
+Performance Strategies, Linear Algebra
+
+## Supported SM Architectures
+
+[SM 5.0 ](https://developer.nvidia.com/cuda-gpus)  [SM 5.2 ](https://developer.nvidia.com/cuda-gpus)  [SM 5.3 ](https://developer.nvidia.com/cuda-gpus)  [SM 6.0 ](https://developer.nvidia.com/cuda-gpus)  [SM 6.1 ](https://developer.nvidia.com/cuda-gpus)  [SM 7.0 ](https://developer.nvidia.com/cuda-gpus)  [SM 7.2 ](https://developer.nvidia.com/cuda-gpus)  [SM 7.5 ](https://developer.nvidia.com/cuda-gpus)  [SM 8.0 ](https://developer.nvidia.com/cuda-gpus)  [SM 8.6 ](https://developer.nvidia.com/cuda-gpus)  [SM 8.7 ](https://developer.nvidia.com/cuda-gpus)  [SM 8.9 ](https://developer.nvidia.com/cuda-gpus)  [SM 9.0 ](https://developer.nvidia.com/cuda-gpus)
+
+## Supported OSes
+
+Linux, Windows
+
+## Supported CPU Architecture
+
+x86_64, armv7l
+
+## CUDA APIs involved
+
+### [CUDA Runtime API](http://docs.nvidia.com/cuda/cuda-runtime-api/index.html)
+cudaMemcpy, cudaMalloc, cudaFree, cudaGetLastError, cudaEventSynchronize, cudaEventRecord, cudaGetDevice, cudaEventDestroy, cudaEventElapsedTime, cudaGetDeviceProperties, cudaEventCreate
+
+## Prerequisites
+
+Download and install the [CUDA Toolkit](https://developer.nvidia.com/cuda-downloads) for your corresponding platform.
+
+## References (for more details)
+
+[whitepaper](./doc/MatrixTranspose.pdf)

cpp/6_Performance/transpose/transpose.cu

@@ -0,0 +1,642 @@
+/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+// -----------------------------------------------------------------------------
+// Transpose
+//
+// This file contains both device and host code for transposing a floating-point
+// matrix.  It performs several transpose kernels, which incrementally improve
+// performance through coalescing, removing shared memory bank conflicts, and
+// eliminating partition camping.  Several of the kernels perform a copy, used
+// to represent the best case performance that a transpose can achieve.
+//
+// Please see the whitepaper in the docs folder of the transpose project for a
+// detailed description of this performance study.
+// -----------------------------------------------------------------------------
+
+#include <cooperative_groups.h>
+
+namespace cg = cooperative_groups;
+// Utilities and system includes
+#include <helper_cuda.h>   // helper for cuda error checking functions
+#include <helper_image.h>  // helper for image and data comparison
+#include <helper_string.h> // helper for string parsing
+
+const char *sSDKsample = "Transpose";
+
+// Each block transposes/copies a tile of TILE_DIM x TILE_DIM elements
+// using TILE_DIM x BLOCK_ROWS threads, so that each thread transposes
+// TILE_DIM/BLOCK_ROWS elements.  TILE_DIM must be an integral multiple of
+// BLOCK_ROWS
+
+#define TILE_DIM   32
+#define BLOCK_ROWS 16
+
+// This sample assumes that MATRIX_SIZE_X = MATRIX_SIZE_Y
+int MATRIX_SIZE_X = 1024;
+int MATRIX_SIZE_Y = 1024;
+int MUL_FACTOR    = TILE_DIM;
+
+#define FLOOR(a, b) (a - (a % b))
+
+// Compute the tile size necessary to illustrate performance cases for SM20+
+// hardware
+int MAX_TILES = (FLOOR(MATRIX_SIZE_X, 512) * FLOOR(MATRIX_SIZE_Y, 512)) / (TILE_DIM * TILE_DIM);
+
+// Number of repetitions used for timing.  Two sets of repetitions are
+// performed: 1) over kernel launches and 2) inside the kernel over just the
+// loads and stores
+
+#define NUM_REPS 100
+
+// -------------------------------------------------------
+// Copies
+// width and height must be integral multiples of TILE_DIM
+// -------------------------------------------------------
+
+__global__ void copy(float *odata, float *idata, int width, int height)
+{
+    int xIndex = blockIdx.x * TILE_DIM + threadIdx.x;
+    int yIndex = blockIdx.y * TILE_DIM + threadIdx.y;
+
+    int index = xIndex + width * yIndex;
+
+    for (int i = 0; i < TILE_DIM; i += BLOCK_ROWS) {
+        odata[index + i * width] = idata[index + i * width];
+    }
+}
+
+__global__ void copySharedMem(float *odata, float *idata, int width, int height)
+{
+    // Handle to thread block group
+    cg::thread_block cta = cg::this_thread_block();
+    __shared__ float tile[TILE_DIM][TILE_DIM];
+
+    int xIndex = blockIdx.x * TILE_DIM + threadIdx.x;
+    int yIndex = blockIdx.y * TILE_DIM + threadIdx.y;
+
+    int index = xIndex + width * yIndex;
+
+    for (int i = 0; i < TILE_DIM; i += BLOCK_ROWS) {
+        if (xIndex < width && yIndex < height) {
+            tile[threadIdx.y + i][threadIdx.x] = idata[index + i * width];
+            tile[threadIdx.y + i][threadIdx.x] = idata[index + i * width];
+        }
+    }
+
+    cg::sync(cta);
+
+    for (int i = 0; i < TILE_DIM; i += BLOCK_ROWS) {
+        if (xIndex < height && yIndex < width) {
+            odata[index + i * width] = tile[threadIdx.y + i][threadIdx.x];
+            odata[index + i * width] = tile[threadIdx.y + i][threadIdx.x];
+        }
+    }
+}
+
+// -------------------------------------------------------
+// Transposes
+// width and height must be integral multiples of TILE_DIM
+// -------------------------------------------------------
+
+__global__ void transposeNaive(float *odata, float *idata, int width, int height)
+{
+    int xIndex = blockIdx.x * TILE_DIM + threadIdx.x;
+    int yIndex = blockIdx.y * TILE_DIM + threadIdx.y;
+
+    int index_in  = xIndex + width * yIndex;
+    int index_out = yIndex + height * xIndex;
+
+    for (int i = 0; i < TILE_DIM; i += BLOCK_ROWS) {
+        odata[index_out + i] = idata[index_in + i * width];
+    }
+}
+
+// coalesced transpose (with bank conflicts)
+
+__global__ void transposeCoalesced(float *odata, float *idata, int width, int height)
+{
+    // Handle to thread block group
+    cg::thread_block cta = cg::this_thread_block();
+    __shared__ float tile[TILE_DIM][TILE_DIM];
+
+    int xIndex   = blockIdx.x * TILE_DIM + threadIdx.x;
+    int yIndex   = blockIdx.y * TILE_DIM + threadIdx.y;
+    int index_in = xIndex + (yIndex)*width;
+
+    xIndex        = blockIdx.y * TILE_DIM + threadIdx.x;
+    yIndex        = blockIdx.x * TILE_DIM + threadIdx.y;
+    int index_out = xIndex + (yIndex)*height;
+
+    for (int i = 0; i < TILE_DIM; i += BLOCK_ROWS) {
+        tile[threadIdx.y + i][threadIdx.x] = idata[index_in + i * width];
+    }
+
+    cg::sync(cta);
+
+    for (int i = 0; i < TILE_DIM; i += BLOCK_ROWS) {
+        odata[index_out + i * height] = tile[threadIdx.x][threadIdx.y + i];
+    }
+}
+
+// Coalesced transpose with no bank conflicts
+
+__global__ void transposeNoBankConflicts(float *odata, float *idata, int width, int height)
+{
+    // Handle to thread block group
+    cg::thread_block cta = cg::this_thread_block();
+    __shared__ float tile[TILE_DIM][TILE_DIM + 1];
+
+    int xIndex   = blockIdx.x * TILE_DIM + threadIdx.x;
+    int yIndex   = blockIdx.y * TILE_DIM + threadIdx.y;
+    int index_in = xIndex + (yIndex)*width;
+
+    xIndex        = blockIdx.y * TILE_DIM + threadIdx.x;
+    yIndex        = blockIdx.x * TILE_DIM + threadIdx.y;
+    int index_out = xIndex + (yIndex)*height;
+
+    for (int i = 0; i < TILE_DIM; i += BLOCK_ROWS) {
+        tile[threadIdx.y + i][threadIdx.x] = idata[index_in + i * width];
+    }
+
+    cg::sync(cta);
+
+    for (int i = 0; i < TILE_DIM; i += BLOCK_ROWS) {
+        odata[index_out + i * height] = tile[threadIdx.x][threadIdx.y + i];
+    }
+}
+
+// Transpose that effectively reorders execution of thread blocks along
+// diagonals of the matrix (also coalesced and has no bank conflicts)
+//
+// Here blockIdx.x is interpreted as the distance along a diagonal and
+// blockIdx.y as corresponding to different diagonals
+//
+// blockIdx_x and blockIdx_y expressions map the diagonal coordinates to the
+// more commonly used cartesian coordinates so that the only changes to the code
+// from the coalesced version are the calculation of the blockIdx_x and
+// blockIdx_y and replacement of blockIdx.x and bloclIdx.y with the subscripted
+// versions in the remaining code
+
+__global__ void transposeDiagonal(float *odata, float *idata, int width, int height)
+{
+    // Handle to thread block group
+    cg::thread_block cta = cg::this_thread_block();
+    __shared__ float tile[TILE_DIM][TILE_DIM + 1];
+
+    int blockIdx_x, blockIdx_y;
+
+    // do diagonal reordering
+    if (width == height) {
+        blockIdx_y = blockIdx.x;
+        blockIdx_x = (blockIdx.x + blockIdx.y) % gridDim.x;
+    }
+    else {
+        int bid    = blockIdx.x + gridDim.x * blockIdx.y;
+        blockIdx_y = bid % gridDim.y;
+        blockIdx_x = ((bid / gridDim.y) + blockIdx_y) % gridDim.x;
+    }
+
+    // from here on the code is same as previous kernel except blockIdx_x replaces
+    // blockIdx.x and similarly for y
+
+    int xIndex   = blockIdx_x * TILE_DIM + threadIdx.x;
+    int yIndex   = blockIdx_y * TILE_DIM + threadIdx.y;
+    int index_in = xIndex + (yIndex)*width;
+
+    xIndex        = blockIdx_y * TILE_DIM + threadIdx.x;
+    yIndex        = blockIdx_x * TILE_DIM + threadIdx.y;
+    int index_out = xIndex + (yIndex)*height;
+
+    for (int i = 0; i < TILE_DIM; i += BLOCK_ROWS) {
+        tile[threadIdx.y + i][threadIdx.x] = idata[index_in + i * width];
+    }
+
+    cg::sync(cta);
+
+    for (int i = 0; i < TILE_DIM; i += BLOCK_ROWS) {
+        odata[index_out + i * height] = tile[threadIdx.x][threadIdx.y + i];
+    }
+}
+
+// --------------------------------------------------------------------
+// Partial transposes
+// NB: the coarse- and fine-grained routines only perform part of a
+//     transpose and will fail the test against the reference solution
+//
+//     They are used to assess performance characteristics of different
+//     components of a full transpose
+// --------------------------------------------------------------------
+
+__global__ void transposeFineGrained(float *odata, float *idata, int width, int height)
+{
+    // Handle to thread block group
+    cg::thread_block cta = cg::this_thread_block();
+    __shared__ float block[TILE_DIM][TILE_DIM + 1];
+
+    int xIndex = blockIdx.x * TILE_DIM + threadIdx.x;
+    int yIndex = blockIdx.y * TILE_DIM + threadIdx.y;
+    int index  = xIndex + (yIndex)*width;
+
+    for (int i = 0; i < TILE_DIM; i += BLOCK_ROWS) {
+        block[threadIdx.y + i][threadIdx.x] = idata[index + i * width];
+    }
+
+    cg::sync(cta);
+
+    for (int i = 0; i < TILE_DIM; i += BLOCK_ROWS) {
+        odata[index + i * height] = block[threadIdx.x][threadIdx.y + i];
+    }
+}
+
+__global__ void transposeCoarseGrained(float *odata, float *idata, int width, int height)
+{
+    // Handle to thread block group
+    cg::thread_block cta = cg::this_thread_block();
+    __shared__ float block[TILE_DIM][TILE_DIM + 1];
+
+    int xIndex   = blockIdx.x * TILE_DIM + threadIdx.x;
+    int yIndex   = blockIdx.y * TILE_DIM + threadIdx.y;
+    int index_in = xIndex + (yIndex)*width;
+
+    xIndex        = blockIdx.y * TILE_DIM + threadIdx.x;
+    yIndex        = blockIdx.x * TILE_DIM + threadIdx.y;
+    int index_out = xIndex + (yIndex)*height;
+
+    for (int i = 0; i < TILE_DIM; i += BLOCK_ROWS) {
+        block[threadIdx.y + i][threadIdx.x] = idata[index_in + i * width];
+    }
+
+    cg::sync(cta);
+
+    for (int i = 0; i < TILE_DIM; i += BLOCK_ROWS) {
+        odata[index_out + i * height] = block[threadIdx.y + i][threadIdx.x];
+    }
+}
+
+// ---------------------
+// host utility routines
+// ---------------------
+
+void computeTransposeGold(float *gold, float *idata, const int size_x, const int size_y)
+{
+    for (int y = 0; y < size_y; ++y) {
+        for (int x = 0; x < size_x; ++x) {
+            gold[(x * size_y) + y] = idata[(y * size_x) + x];
+        }
+    }
+}
+
+void getParams(int argc, char **argv, cudaDeviceProp &deviceProp, int &size_x, int &size_y, int max_tile_dim)
+{
+    // set matrix size (if (x,y) dim of matrix is not square, then this will have
+    // to be modified
+    if (checkCmdLineFlag(argc, (const char **)argv, "dimX")) {
+        size_x = getCmdLineArgumentInt(argc, (const char **)argv, "dimX");
+
+        if (size_x > max_tile_dim) {
+            printf("> MatrixSize X = %d is greater than the recommended size = %d\n", size_x, max_tile_dim);
+        }
+        else {
+            printf("> MatrixSize X = %d\n", size_x);
+        }
+    }
+    else {
+        size_x = max_tile_dim;
+        size_x = FLOOR(size_x, 512);
+    }
+
+    if (checkCmdLineFlag(argc, (const char **)argv, "dimY")) {
+        size_y = getCmdLineArgumentInt(argc, (const char **)argv, "dimY");
+
+        if (size_y > max_tile_dim) {
+            printf("> MatrixSize Y = %d is greater than the recommended size = %d\n", size_y, max_tile_dim);
+        }
+        else {
+            printf("> MatrixSize Y = %d\n", size_y);
+        }
+    }
+    else {
+        size_y = max_tile_dim;
+        size_y = FLOOR(size_y, 512);
+    }
+}
+
+void showHelp()
+{
+    printf("\n%s : Command line options\n", sSDKsample);
+    printf("\t-device=n          (where n=0,1,2.... for the GPU device)\n\n");
+    printf("> The default matrix size can be overridden with these parameters\n");
+    printf("\t-dimX=row_dim_size (matrix row    dimensions)\n");
+    printf("\t-dimY=col_dim_size (matrix column dimensions)\n");
+}
+
+// ----
+// main
+// ----
+
+int main(int argc, char **argv)
+{
+    // Start logs
+    printf("%s Starting...\n\n", sSDKsample);
+
+    if (checkCmdLineFlag(argc, (const char **)argv, "help")) {
+        showHelp();
+        return 0;
+    }
+
+    int            devID = findCudaDevice(argc, (const char **)argv);
+    cudaDeviceProp deviceProp;
+
+    // get number of SMs on this GPU
+    checkCudaErrors(cudaGetDevice(&devID));
+    checkCudaErrors(cudaGetDeviceProperties(&deviceProp, devID));
+
+    // compute the scaling factor (for GPUs with fewer MPs)
+    float scale_factor, total_tiles;
+    scale_factor = max(
+        (192.0f / (_ConvertSMVer2Cores(deviceProp.major, deviceProp.minor) * (float)deviceProp.multiProcessorCount)),
+        1.0f);
+
+    printf("> Device %d: \"%s\"\n", devID, deviceProp.name);
+    printf("> SM Capability %d.%d detected:\n", deviceProp.major, deviceProp.minor);
+
+    // Calculate number of tiles we will run for the Matrix Transpose performance
+    // tests
+    int size_x, size_y, max_matrix_dim, matrix_size_test;
+
+    matrix_size_test = 512; // we round down max_matrix_dim for this perf test
+    total_tiles      = (float)MAX_TILES / scale_factor;
+
+    max_matrix_dim = FLOOR((int)(floor(sqrt(total_tiles)) * TILE_DIM), matrix_size_test);
+
+    // This is the minimum size allowed
+    if (max_matrix_dim == 0) {
+        max_matrix_dim = matrix_size_test;
+    }
+
+    printf("> [%s] has %d MP(s) x %d (Cores/MP) = %d (Cores)\n",
+           deviceProp.name,
+           deviceProp.multiProcessorCount,
+           _ConvertSMVer2Cores(deviceProp.major, deviceProp.minor),
+           _ConvertSMVer2Cores(deviceProp.major, deviceProp.minor) * deviceProp.multiProcessorCount);
+
+    printf("> Compute performance scaling factor = %4.2f\n", scale_factor);
+
+    // Extract parameters if there are any, command line -dimx and -dimy can
+    // override any of these settings
+    getParams(argc, argv, deviceProp, size_x, size_y, max_matrix_dim);
+
+    if (size_x != size_y) {
+        printf("\n[%s] does not support non-square matrices (row_dim_size(%d) != "
+               "col_dim_size(%d))\nExiting...\n\n",
+               sSDKsample,
+               size_x,
+               size_y);
+        exit(EXIT_FAILURE);
+    }
+
+    if (size_x % TILE_DIM != 0 || size_y % TILE_DIM != 0) {
+        printf("[%s] Matrix size must be integral multiple of tile "
+               "size\nExiting...\n\n",
+               sSDKsample);
+        exit(EXIT_FAILURE);
+    }
+
+    // kernel pointer and descriptor
+    void (*kernel)(float *, float *, int, int);
+    const char *kernelName;
+
+    // execution configuration parameters
+    dim3 grid(size_x / TILE_DIM, size_y / TILE_DIM), threads(TILE_DIM, BLOCK_ROWS);
+
+    if (grid.x < 1 || grid.y < 1) {
+        printf("[%s] grid size computation incorrect in test \nExiting...\n\n", sSDKsample);
+        exit(EXIT_FAILURE);
+    }
+
+    // CUDA events
+    cudaEvent_t start, stop;
+
+    // size of memory required to store the matrix
+    size_t mem_size = static_cast<size_t>(sizeof(float) * size_x * size_y);
+
+    if (2 * mem_size > deviceProp.totalGlobalMem) {
+        printf("Input matrix size is larger than the available device memory!\n");
+        printf("Please choose a smaller size matrix\n");
+        exit(EXIT_FAILURE);
+    }
+
+    // allocate host memory
+    float *h_idata       = (float *)malloc(mem_size);
+    float *h_odata       = (float *)malloc(mem_size);
+    float *transposeGold = (float *)malloc(mem_size);
+    float *gold;
+
+    // allocate device memory
+    float *d_idata, *d_odata;
+    checkCudaErrors(cudaMalloc((void **)&d_idata, mem_size));
+    checkCudaErrors(cudaMalloc((void **)&d_odata, mem_size));
+
+    // initialize host data
+    for (int i = 0; i < (size_x * size_y); ++i) {
+        h_idata[i] = (float)i;
+    }
+
+    // copy host data to device
+    checkCudaErrors(cudaMemcpy(d_idata, h_idata, mem_size, cudaMemcpyHostToDevice));
+
+    // Compute reference transpose solution
+    computeTransposeGold(transposeGold, h_idata, size_x, size_y);
+
+    // print out common data for all kernels
+    printf("\nMatrix size: %dx%d (%dx%d tiles), tile size: %dx%d, block size: "
+           "%dx%d\n\n",
+           size_x,
+           size_y,
+           size_x / TILE_DIM,
+           size_y / TILE_DIM,
+           TILE_DIM,
+           TILE_DIM,
+           TILE_DIM,
+           BLOCK_ROWS);
+
+    // initialize events
+    checkCudaErrors(cudaEventCreate(&start));
+    checkCudaErrors(cudaEventCreate(&stop));
+
+    //
+    // loop over different kernels
+    //
+
+    bool success = true;
+
+    for (int k = 0; k < 8; k++) {
+        // set kernel pointer
+        switch (k) {
+        case 0:
+            kernel     = &copy;
+            kernelName = "simple copy       ";
+            break;
+
+        case 1:
+            kernel     = &copySharedMem;
+            kernelName = "shared memory copy";
+            break;
+
+        case 2:
+            kernel     = &transposeNaive;
+            kernelName = "naive             ";
+            break;
+
+        case 3:
+            kernel     = &transposeCoalesced;
+            kernelName = "coalesced         ";
+            break;
+
+        case 4:
+            kernel     = &transposeNoBankConflicts;
+            kernelName = "optimized         ";
+            break;
+
+        case 5:
+            kernel     = &transposeCoarseGrained;
+            kernelName = "coarse-grained    ";
+            break;
+
+        case 6:
+            kernel     = &transposeFineGrained;
+            kernelName = "fine-grained      ";
+            break;
+
+        case 7:
+            kernel     = &transposeDiagonal;
+            kernelName = "diagonal          ";
+            break;
+        }
+
+        // set reference solution
+        if (kernel == &copy || kernel == &copySharedMem) {
+            gold = h_idata;
+        }
+        else if (kernel == &transposeCoarseGrained || kernel == &transposeFineGrained) {
+            gold = h_odata; // fine- and coarse-grained kernels are not full
+                            // transposes, so bypass check
+        }
+        else {
+            gold = transposeGold;
+        }
+
+        // Clear error status
+        checkCudaErrors(cudaGetLastError());
+
+        // warmup to avoid timing startup
+        kernel<<<grid, threads>>>(d_odata, d_idata, size_x, size_y);
+
+        // take measurements for loop over kernel launches
+        checkCudaErrors(cudaEventRecord(start, 0));
+
+        for (int i = 0; i < NUM_REPS; i++) {
+            kernel<<<grid, threads>>>(d_odata, d_idata, size_x, size_y);
+            // Ensure no launch failure
+            checkCudaErrors(cudaGetLastError());
+        }
+
+        checkCudaErrors(cudaEventRecord(stop, 0));
+        checkCudaErrors(cudaEventSynchronize(stop));
+        float kernelTime;
+        checkCudaErrors(cudaEventElapsedTime(&kernelTime, start, stop));
+
+        checkCudaErrors(cudaMemcpy(h_odata, d_odata, mem_size, cudaMemcpyDeviceToHost));
+        bool res = compareData(gold, h_odata, size_x * size_y, 0.01f, 0.0f);
+
+        if (res == false) {
+            printf("*** %s kernel FAILED ***\n", kernelName);
+            success = false;
+        }
+
+        // take measurements for loop inside kernel
+        checkCudaErrors(cudaMemcpy(h_odata, d_odata, mem_size, cudaMemcpyDeviceToHost));
+        res = compareData(gold, h_odata, size_x * size_y, 0.01f, 0.0f);
+
+        if (res == false) {
+            printf("*** %s kernel FAILED ***\n", kernelName);
+            success = false;
+        }
+
+        // report effective bandwidths
+        float kernelBandwidth = 2.0f * 1000.0f * mem_size / (1024 * 1024 * 1024) / (kernelTime / NUM_REPS);
+        printf("transpose %s, Throughput = %.4f GB/s, Time = %.5f ms, Size = %u fp32 "
+               "elements, NumDevsUsed = %u, Workgroup = %u\n",
+               kernelName,
+               kernelBandwidth,
+               kernelTime / NUM_REPS,
+               (size_x * size_y),
+               1,
+               TILE_DIM * BLOCK_ROWS);
+
+        // Reset d_odata to zero before starting the next loop iteration to avoid
+        // carrying over results from previous kernels. Without this reset, residual
+        // data from a prior kernel (e.g., 'copy') could make a subsequent
+        // kernel (e.g., 'copySharedMem') appear correct even if it performs no work,
+        // leading to false positives in compareData.
+        for (int i = 0; i < (size_x * size_y); ++i) {
+            h_odata[i] = 0;
+        }
+        // copy host data to device
+        checkCudaErrors(cudaMemcpy(d_odata, h_odata, mem_size, cudaMemcpyHostToDevice));
+
+        // Reset d_odata to zero before starting the next loop iteration to avoid
+        // carrying over results from previous kernels. Without this reset, residual
+        // data from a prior kernel (e.g., 'copy') could make a subsequent
+        // kernel (e.g., 'copySharedMem') appear correct even if it performs no work,
+        // leading to false positives in compareData.
+        for (int i = 0; i < (size_x * size_y); ++i) {
+            h_odata[i] = 0;
+        }
+        // copy host data to device
+        checkCudaErrors(cudaMemcpy(d_odata, h_odata, mem_size, cudaMemcpyHostToDevice));
+    }
+
+    // cleanup
+    free(h_idata);
+    free(h_odata);
+    free(transposeGold);
+    cudaFree(d_idata);
+    cudaFree(d_odata);
+
+    checkCudaErrors(cudaEventDestroy(start));
+    checkCudaErrors(cudaEventDestroy(stop));
+
+    if (!success) {
+        printf("Test failed!\n");
+        exit(EXIT_FAILURE);
+    }
+
+    printf("Test passed\n");
+    exit(EXIT_SUCCESS);
+}