add and update samples for CUDA 11.5

Samples/BlackScholes/.vscode/c_cpp_properties.json

@@ -0,0 +1,18 @@
+{
+    "configurations": [
+        {
+            "name": "Linux",
+            "includePath": [
+                "${workspaceFolder}/**",
+                "${workspaceFolder}/../../Common"
+            ],
+            "defines": [],
+            "compilerPath": "/usr/local/cuda/bin/nvcc",
+            "cStandard": "gnu17",
+            "cppStandard": "gnu++14",
+            "intelliSenseMode": "linux-gcc-x64",
+            "configurationProvider": "ms-vscode.makefile-tools"
+        }
+    ],
+    "version": 4
+}

Samples/BlackScholes/.vscode/extensions.json

@@ -0,0 +1,7 @@
+{
+    "recommendations": [
+        "nvidia.nsight-vscode-edition",
+        "ms-vscode.cpptools",
+        "ms-vscode.makefile-tools"
+    ]
+}

Samples/BlackScholes/.vscode/launch.json

@@ -0,0 +1,10 @@
+{
+    "configurations": [
+        {
+            "name": "CUDA C++: Launch",
+            "type": "cuda-gdb",
+            "request": "launch",
+            "program": "${workspaceFolder}/BlackScholes"
+        }
+    ]
+}

Samples/BlackScholes/.vscode/tasks.json

@@ -0,0 +1,15 @@
+{
+    "version": "2.0.0",
+    "tasks": [
+        {
+            "label": "sample",
+            "type": "shell",
+            "command": "make dbg=1",
+            "problemMatcher": ["$nvcc"],
+            "group": {
+                "kind": "build",
+                "isDefault": true
+            }
+        }
+    ]
+}

Samples/BlackScholes/BlackScholes.cu

@@ -0,0 +1,243 @@
+/* Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * This sample evaluates fair call and put prices for a
+ * given set of European options by Black-Scholes formula.
+ * See supplied whitepaper for more explanations.
+ */
+
+#include <helper_functions.h>  // helper functions for string parsing
+#include <helper_cuda.h>  // helper functions CUDA error checking and initialization
+
+////////////////////////////////////////////////////////////////////////////////
+// Process an array of optN options on CPU
+////////////////////////////////////////////////////////////////////////////////
+extern "C" void BlackScholesCPU(float *h_CallResult, float *h_PutResult,
+                                float *h_StockPrice, float *h_OptionStrike,
+                                float *h_OptionYears, float Riskfree,
+                                float Volatility, int optN);
+
+////////////////////////////////////////////////////////////////////////////////
+// Process an array of OptN options on GPU
+////////////////////////////////////////////////////////////////////////////////
+#include "BlackScholes_kernel.cuh"
+
+////////////////////////////////////////////////////////////////////////////////
+// Helper function, returning uniformly distributed
+// random float in [low, high] range
+////////////////////////////////////////////////////////////////////////////////
+float RandFloat(float low, float high) {
+  float t = (float)rand() / (float)RAND_MAX;
+  return (1.0f - t) * low + t * high;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// Data configuration
+////////////////////////////////////////////////////////////////////////////////
+const int OPT_N = 4000000;
+const int NUM_ITERATIONS = 512;
+
+const int OPT_SZ = OPT_N * sizeof(float);
+const float RISKFREE = 0.02f;
+const float VOLATILITY = 0.30f;
+
+#define DIV_UP(a, b) (((a) + (b)-1) / (b))
+
+////////////////////////////////////////////////////////////////////////////////
+// Main program
+////////////////////////////////////////////////////////////////////////////////
+int main(int argc, char **argv) {
+  // Start logs
+  printf("[%s] - Starting...\n", argv[0]);
+
+  //'h_' prefix - CPU (host) memory space
+  float
+      // Results calculated by CPU for reference
+      *h_CallResultCPU,
+      *h_PutResultCPU,
+      // CPU copy of GPU results
+      *h_CallResultGPU, *h_PutResultGPU,
+      // CPU instance of input data
+      *h_StockPrice, *h_OptionStrike, *h_OptionYears;
+
+  //'d_' prefix - GPU (device) memory space
+  float
+      // Results calculated by GPU
+      *d_CallResult,
+      *d_PutResult,
+      // GPU instance of input data
+      *d_StockPrice, *d_OptionStrike, *d_OptionYears;
+
+  double delta, ref, sum_delta, sum_ref, max_delta, L1norm, gpuTime;
+
+  StopWatchInterface *hTimer = NULL;
+  int i;
+
+  findCudaDevice(argc, (const char **)argv);
+
+  sdkCreateTimer(&hTimer);
+
+  printf("Initializing data...\n");
+  printf("...allocating CPU memory for options.\n");
+  h_CallResultCPU = (float *)malloc(OPT_SZ);
+  h_PutResultCPU = (float *)malloc(OPT_SZ);
+  h_CallResultGPU = (float *)malloc(OPT_SZ);
+  h_PutResultGPU = (float *)malloc(OPT_SZ);
+  h_StockPrice = (float *)malloc(OPT_SZ);
+  h_OptionStrike = (float *)malloc(OPT_SZ);
+  h_OptionYears = (float *)malloc(OPT_SZ);
+
+  printf("...allocating GPU memory for options.\n");
+  checkCudaErrors(cudaMalloc((void **)&d_CallResult, OPT_SZ));
+  checkCudaErrors(cudaMalloc((void **)&d_PutResult, OPT_SZ));
+  checkCudaErrors(cudaMalloc((void **)&d_StockPrice, OPT_SZ));
+  checkCudaErrors(cudaMalloc((void **)&d_OptionStrike, OPT_SZ));
+  checkCudaErrors(cudaMalloc((void **)&d_OptionYears, OPT_SZ));
+
+  printf("...generating input data in CPU mem.\n");
+  srand(5347);
+
+  // Generate options set
+  for (i = 0; i < OPT_N; i++) {
+    h_CallResultCPU[i] = 0.0f;
+    h_PutResultCPU[i] = -1.0f;
+    h_StockPrice[i] = RandFloat(5.0f, 30.0f);
+    h_OptionStrike[i] = RandFloat(1.0f, 100.0f);
+    h_OptionYears[i] = RandFloat(0.25f, 10.0f);
+  }
+
+  printf("...copying input data to GPU mem.\n");
+  // Copy options data to GPU memory for further processing
+  checkCudaErrors(
+      cudaMemcpy(d_StockPrice, h_StockPrice, OPT_SZ, cudaMemcpyHostToDevice));
+  checkCudaErrors(cudaMemcpy(d_OptionStrike, h_OptionStrike, OPT_SZ,
+                             cudaMemcpyHostToDevice));
+  checkCudaErrors(
+      cudaMemcpy(d_OptionYears, h_OptionYears, OPT_SZ, cudaMemcpyHostToDevice));
+  printf("Data init done.\n\n");
+
+  printf("Executing Black-Scholes GPU kernel (%i iterations)...\n",
+         NUM_ITERATIONS);
+  checkCudaErrors(cudaDeviceSynchronize());
+  sdkResetTimer(&hTimer);
+  sdkStartTimer(&hTimer);
+
+  for (i = 0; i < NUM_ITERATIONS; i++) {
+    BlackScholesGPU<<<DIV_UP((OPT_N / 2), 128), 128 /*480, 128*/>>>(
+        (float2 *)d_CallResult, (float2 *)d_PutResult, (float2 *)d_StockPrice,
+        (float2 *)d_OptionStrike, (float2 *)d_OptionYears, RISKFREE, VOLATILITY,
+        OPT_N);
+    getLastCudaError("BlackScholesGPU() execution failed\n");
+  }
+
+  checkCudaErrors(cudaDeviceSynchronize());
+  sdkStopTimer(&hTimer);
+  gpuTime = sdkGetTimerValue(&hTimer) / NUM_ITERATIONS;
+
+  // Both call and put is calculated
+  printf("Options count             : %i     \n", 2 * OPT_N);
+  printf("BlackScholesGPU() time    : %f msec\n", gpuTime);
+  printf("Effective memory bandwidth: %f GB/s\n",
+         ((double)(5 * OPT_N * sizeof(float)) * 1E-9) / (gpuTime * 1E-3));
+  printf("Gigaoptions per second    : %f     \n\n",
+         ((double)(2 * OPT_N) * 1E-9) / (gpuTime * 1E-3));
+
+  printf(
+      "BlackScholes, Throughput = %.4f GOptions/s, Time = %.5f s, Size = %u "
+      "options, NumDevsUsed = %u, Workgroup = %u\n",
+      (((double)(2.0 * OPT_N) * 1.0E-9) / (gpuTime * 1.0E-3)), gpuTime * 1e-3,
+      (2 * OPT_N), 1, 128);
+
+  printf("\nReading back GPU results...\n");
+  // Read back GPU results to compare them to CPU results
+  checkCudaErrors(cudaMemcpy(h_CallResultGPU, d_CallResult, OPT_SZ,
+                             cudaMemcpyDeviceToHost));
+  checkCudaErrors(
+      cudaMemcpy(h_PutResultGPU, d_PutResult, OPT_SZ, cudaMemcpyDeviceToHost));
+
+  printf("Checking the results...\n");
+  printf("...running CPU calculations.\n\n");
+  // Calculate options values on CPU
+  BlackScholesCPU(h_CallResultCPU, h_PutResultCPU, h_StockPrice, h_OptionStrike,
+                  h_OptionYears, RISKFREE, VOLATILITY, OPT_N);
+
+  printf("Comparing the results...\n");
+  // Calculate max absolute difference and L1 distance
+  // between CPU and GPU results
+  sum_delta = 0;
+  sum_ref = 0;
+  max_delta = 0;
+
+  for (i = 0; i < OPT_N; i++) {
+    ref = h_CallResultCPU[i];
+    delta = fabs(h_CallResultCPU[i] - h_CallResultGPU[i]);
+
+    if (delta > max_delta) {
+      max_delta = delta;
+    }
+
+    sum_delta += delta;
+    sum_ref += fabs(ref);
+  }
+
+  L1norm = sum_delta / sum_ref;
+  printf("L1 norm: %E\n", L1norm);
+  printf("Max absolute error: %E\n\n", max_delta);
+
+  printf("Shutting down...\n");
+  printf("...releasing GPU memory.\n");
+  checkCudaErrors(cudaFree(d_OptionYears));
+  checkCudaErrors(cudaFree(d_OptionStrike));
+  checkCudaErrors(cudaFree(d_StockPrice));
+  checkCudaErrors(cudaFree(d_PutResult));
+  checkCudaErrors(cudaFree(d_CallResult));
+
+  printf("...releasing CPU memory.\n");
+  free(h_OptionYears);
+  free(h_OptionStrike);
+  free(h_StockPrice);
+  free(h_PutResultGPU);
+  free(h_CallResultGPU);
+  free(h_PutResultCPU);
+  free(h_CallResultCPU);
+  sdkDeleteTimer(&hTimer);
+  printf("Shutdown done.\n");
+
+  printf("\n[BlackScholes] - Test Summary\n");
+
+  if (L1norm > 1e-6) {
+    printf("Test failed!\n");
+    exit(EXIT_FAILURE);
+  }
+
+  printf(
+      "\nNOTE: The CUDA Samples are not meant for performance measurements. "
+      "Results may vary when GPU Boost is enabled.\n\n");
+  printf("Test passed\n");
+  exit(EXIT_SUCCESS);
+}

Samples/BlackScholes/BlackScholes_gold.cpp

@@ -0,0 +1,86 @@
+/* Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <math.h>
+
+////////////////////////////////////////////////////////////////////////////////
+// Polynomial approximation of cumulative normal distribution function
+////////////////////////////////////////////////////////////////////////////////
+static double CND(double d) {
+  const double A1 = 0.31938153;
+  const double A2 = -0.356563782;
+  const double A3 = 1.781477937;
+  const double A4 = -1.821255978;
+  const double A5 = 1.330274429;
+  const double RSQRT2PI = 0.39894228040143267793994605993438;
+
+  double K = 1.0 / (1.0 + 0.2316419 * fabs(d));
+
+  double cnd = RSQRT2PI * exp(-0.5 * d * d) *
+               (K * (A1 + K * (A2 + K * (A3 + K * (A4 + K * A5)))));
+
+  if (d > 0) cnd = 1.0 - cnd;
+
+  return cnd;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// Black-Scholes formula for both call and put
+////////////////////////////////////////////////////////////////////////////////
+static void BlackScholesBodyCPU(float &callResult, float &putResult,
+                                float Sf,  // Stock price
+                                float Xf,  // Option strike
+                                float Tf,  // Option years
+                                float Rf,  // Riskless rate
+                                float Vf  // Volatility rate
+                                ) {
+  double S = Sf, X = Xf, T = Tf, R = Rf, V = Vf;
+
+  double sqrtT = sqrt(T);
+  double d1 = (log(S / X) + (R + 0.5 * V * V) * T) / (V * sqrtT);
+  double d2 = d1 - V * sqrtT;
+  double CNDD1 = CND(d1);
+  double CNDD2 = CND(d2);
+
+  // Calculate Call and Put simultaneously
+  double expRT = exp(-R * T);
+  callResult = (float)(S * CNDD1 - X * expRT * CNDD2);
+  putResult = (float)(X * expRT * (1.0 - CNDD2) - S * (1.0 - CNDD1));
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// Process an array of optN options
+////////////////////////////////////////////////////////////////////////////////
+extern "C" void BlackScholesCPU(float *h_CallResult, float *h_PutResult,
+                                float *h_StockPrice, float *h_OptionStrike,
+                                float *h_OptionYears, float Riskfree,
+                                float Volatility, int optN) {
+  for (int opt = 0; opt < optN; opt++)
+    BlackScholesBodyCPU(h_CallResult[opt], h_PutResult[opt], h_StockPrice[opt],
+                        h_OptionStrike[opt], h_OptionYears[opt], Riskfree,
+                        Volatility);
+}

Samples/BlackScholes/BlackScholes_kernel.cuh

@@ -0,0 +1,106 @@
+/* Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+////////////////////////////////////////////////////////////////////////////////
+// Polynomial approximation of cumulative normal distribution function
+////////////////////////////////////////////////////////////////////////////////
+__device__ inline float cndGPU(float d) {
+  const float A1 = 0.31938153f;
+  const float A2 = -0.356563782f;
+  const float A3 = 1.781477937f;
+  const float A4 = -1.821255978f;
+  const float A5 = 1.330274429f;
+  const float RSQRT2PI = 0.39894228040143267793994605993438f;
+
+  float K = __fdividef(1.0f, (1.0f + 0.2316419f * fabsf(d)));
+
+  float cnd = RSQRT2PI * __expf(-0.5f * d * d) *
+              (K * (A1 + K * (A2 + K * (A3 + K * (A4 + K * A5)))));
+
+  if (d > 0) cnd = 1.0f - cnd;
+
+  return cnd;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// Black-Scholes formula for both call and put
+////////////////////////////////////////////////////////////////////////////////
+__device__ inline void BlackScholesBodyGPU(float &CallResult, float &PutResult,
+                                           float S,  // Stock price
+                                           float X,  // Option strike
+                                           float T,  // Option years
+                                           float R,  // Riskless rate
+                                           float V  // Volatility rate
+                                           ) {
+  float sqrtT, expRT;
+  float d1, d2, CNDD1, CNDD2;
+
+  sqrtT = __fdividef(1.0F, rsqrtf(T));
+  d1 = __fdividef(__logf(S / X) + (R + 0.5f * V * V) * T, V * sqrtT);
+  d2 = d1 - V * sqrtT;
+
+  CNDD1 = cndGPU(d1);
+  CNDD2 = cndGPU(d2);
+
+  // Calculate Call and Put simultaneously
+  expRT = __expf(-R * T);
+  CallResult = S * CNDD1 - X * expRT * CNDD2;
+  PutResult = X * expRT * (1.0f - CNDD2) - S * (1.0f - CNDD1);
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// Process an array of optN options on GPU
+////////////////////////////////////////////////////////////////////////////////
+__launch_bounds__(128) __global__
+    void BlackScholesGPU(float2 *__restrict d_CallResult,
+                         float2 *__restrict d_PutResult,
+                         float2 *__restrict d_StockPrice,
+                         float2 *__restrict d_OptionStrike,
+                         float2 *__restrict d_OptionYears, float Riskfree,
+                         float Volatility, int optN) {
+  ////Thread index
+  // const int      tid = blockDim.x * blockIdx.x + threadIdx.x;
+  ////Total number of threads in execution grid
+  // const int THREAD_N = blockDim.x * gridDim.x;
+
+  const int opt = blockDim.x * blockIdx.x + threadIdx.x;
+
+  // Calculating 2 options per thread to increase ILP (instruction level
+  // parallelism)
+  if (opt < (optN / 2)) {
+    float callResult1, callResult2;
+    float putResult1, putResult2;
+    BlackScholesBodyGPU(callResult1, putResult1, d_StockPrice[opt].x,
+                        d_OptionStrike[opt].x, d_OptionYears[opt].x, Riskfree,
+                        Volatility);
+    BlackScholesBodyGPU(callResult2, putResult2, d_StockPrice[opt].y,
+                        d_OptionStrike[opt].y, d_OptionYears[opt].y, Riskfree,
+                        Volatility);
+    d_CallResult[opt] = make_float2(callResult1, callResult2);
+    d_PutResult[opt] = make_float2(putResult1, putResult2);
+  }
+}

Samples/BlackScholes/BlackScholes_vs2017.sln

@@ -0,0 +1,20 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio 2017
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "BlackScholes", "BlackScholes_vs2017.vcxproj", "{997E0757-EA74-4A4E-A0FC-47D8C8831A15}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|x64 = Debug|x64
+		Release|x64 = Release|x64
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Debug|x64.ActiveCfg = Debug|x64
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Debug|x64.Build.0 = Debug|x64
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Release|x64.ActiveCfg = Release|x64
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Release|x64.Build.0 = Release|x64
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+EndGlobal

Samples/BlackScholes/BlackScholes_vs2017.vcxproj

@@ -0,0 +1,114 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <CUDAPropsPath Condition="'$(CUDAPropsPath)'==''">$(VCTargetsPath)\BuildCustomizations</CUDAPropsPath>
+  </PropertyGroup>
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Debug|x64">
+      <Configuration>Debug</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|x64">
+      <Configuration>Release</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>{997E0757-EA74-4A4E-A0FC-47D8C8831A15}</ProjectGuid>
+    <RootNamespace>BlackScholes_vs2017</RootNamespace>
+    <ProjectName>BlackScholes</ProjectName>
+    <CudaToolkitCustomDir />
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(WindowsTargetPlatformVersion)'==''">
+    <LatestTargetPlatformVersion>$([Microsoft.Build.Utilities.ToolLocationHelper]::GetLatestSDKTargetPlatformVersion('Windows', '10.0'))</LatestTargetPlatformVersion>
+    <WindowsTargetPlatformVersion Condition="'$(WindowsTargetPlatformVersion)' == ''">$(LatestTargetPlatformVersion)</WindowsTargetPlatformVersion>
+    <TargetPlatformVersion>$(WindowsTargetPlatformVersion)</TargetPlatformVersion>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <PropertyGroup>
+    <ConfigurationType>Application</ConfigurationType>
+    <CharacterSet>MultiByte</CharacterSet>
+    <PlatformToolset>v141</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)'=='Debug'">
+    <UseDebugLibraries>true</UseDebugLibraries>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)'=='Release'">
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <ImportGroup Label="ExtensionSettings">
+    <Import Project="$(CUDAPropsPath)\CUDA 11.5.props" />
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets">
+    <Import Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" />
+  </ImportGroup>
+  <PropertyGroup Label="UserMacros" />
+  <PropertyGroup>
+    <IntDir>$(Platform)/$(Configuration)/</IntDir>
+    <IncludePath>$(IncludePath)</IncludePath>
+    <CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRules />
+    <CodeAnalysisRuleAssemblies />
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Platform)'=='x64'">
+    <OutDir>../../bin/win64/$(Configuration)/</OutDir>
+  </PropertyGroup>
+  <ItemDefinitionGroup>
+    <ClCompile>
+      <WarningLevel>Level3</WarningLevel>
+      <PreprocessorDefinitions>WIN32;_MBCS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <AdditionalIncludeDirectories>./;$(CudaToolkitDir)/include;../../Common;</AdditionalIncludeDirectories>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <AdditionalDependencies>cudart_static.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalLibraryDirectories>$(CudaToolkitLibDir);</AdditionalLibraryDirectories>
+      <OutputFile>$(OutDir)/BlackScholes.exe</OutputFile>
+    </Link>
+    <CudaCompile>
+      <CodeGeneration>compute_35,sm_35;compute_37,sm_37;compute_50,sm_50;compute_52,sm_52;compute_60,sm_60;compute_61,sm_61;compute_70,sm_70;compute_75,sm_75;compute_80,sm_80;compute_86,sm_86;</CodeGeneration>
+      <AdditionalOptions>-Xcompiler "/wd 4819" %(AdditionalOptions)</AdditionalOptions>
+      <Include>./;../../Common</Include>
+      <Defines>WIN32</Defines>
+      <AdditionalOptions>--threads 0</AdditionalOptions>
+    </CudaCompile>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)'=='Debug'">
+    <ClCompile>
+      <Optimization>Disabled</Optimization>
+      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <LinkTimeCodeGeneration>Default</LinkTimeCodeGeneration>
+    </Link>
+    <CudaCompile>
+      <Runtime>MTd</Runtime>
+      <TargetMachinePlatform>64</TargetMachinePlatform>
+    </CudaCompile>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)'=='Release'">
+    <ClCompile>
+      <Optimization>MaxSpeed</Optimization>
+      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <GenerateDebugInformation>false</GenerateDebugInformation>
+      <LinkTimeCodeGeneration>UseLinkTimeCodeGeneration</LinkTimeCodeGeneration>
+    </Link>
+    <CudaCompile>
+      <Runtime>MT</Runtime>
+      <TargetMachinePlatform>64</TargetMachinePlatform>
+    </CudaCompile>
+  </ItemDefinitionGroup>
+  <ItemGroup>
+    <CudaCompile Include="BlackScholes.cu" />
+    <ClCompile Include="BlackScholes_gold.cpp" />
+    <None Include="BlackScholes_kernel.cuh" />
+  </ItemGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <ImportGroup Label="ExtensionTargets">
+    <Import Project="$(CUDAPropsPath)\CUDA 11.5.targets" />
+  </ImportGroup>
+</Project>

Samples/BlackScholes/BlackScholes_vs2019.sln

@@ -0,0 +1,20 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio 2019
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "BlackScholes", "BlackScholes_vs2019.vcxproj", "{997E0757-EA74-4A4E-A0FC-47D8C8831A15}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|x64 = Debug|x64
+		Release|x64 = Release|x64
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Debug|x64.ActiveCfg = Debug|x64
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Debug|x64.Build.0 = Debug|x64
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Release|x64.ActiveCfg = Release|x64
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Release|x64.Build.0 = Release|x64
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+EndGlobal

Samples/BlackScholes/BlackScholes_vs2019.vcxproj

@@ -0,0 +1,110 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <CUDAPropsPath Condition="'$(CUDAPropsPath)'==''">$(VCTargetsPath)\BuildCustomizations</CUDAPropsPath>
+  </PropertyGroup>
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Debug|x64">
+      <Configuration>Debug</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|x64">
+      <Configuration>Release</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>{997E0757-EA74-4A4E-A0FC-47D8C8831A15}</ProjectGuid>
+    <RootNamespace>BlackScholes_vs2019</RootNamespace>
+    <ProjectName>BlackScholes</ProjectName>
+    <CudaToolkitCustomDir />
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <PropertyGroup>
+    <ConfigurationType>Application</ConfigurationType>
+    <CharacterSet>MultiByte</CharacterSet>
+    <PlatformToolset>v142</PlatformToolset>
+    <WindowsTargetPlatformVersion>10.0</WindowsTargetPlatformVersion>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)'=='Debug'">
+    <UseDebugLibraries>true</UseDebugLibraries>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)'=='Release'">
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <ImportGroup Label="ExtensionSettings">
+    <Import Project="$(CUDAPropsPath)\CUDA 11.5.props" />
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets">
+    <Import Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" />
+  </ImportGroup>
+  <PropertyGroup Label="UserMacros" />
+  <PropertyGroup>
+    <IntDir>$(Platform)/$(Configuration)/</IntDir>
+    <IncludePath>$(IncludePath)</IncludePath>
+    <CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRules />
+    <CodeAnalysisRuleAssemblies />
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Platform)'=='x64'">
+    <OutDir>../../bin/win64/$(Configuration)/</OutDir>
+  </PropertyGroup>
+  <ItemDefinitionGroup>
+    <ClCompile>
+      <WarningLevel>Level3</WarningLevel>
+      <PreprocessorDefinitions>WIN32;_MBCS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <AdditionalIncludeDirectories>./;$(CudaToolkitDir)/include;../../Common;</AdditionalIncludeDirectories>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <AdditionalDependencies>cudart_static.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalLibraryDirectories>$(CudaToolkitLibDir);</AdditionalLibraryDirectories>
+      <OutputFile>$(OutDir)/BlackScholes.exe</OutputFile>
+    </Link>
+    <CudaCompile>
+      <CodeGeneration>compute_35,sm_35;compute_37,sm_37;compute_50,sm_50;compute_52,sm_52;compute_60,sm_60;compute_61,sm_61;compute_70,sm_70;compute_75,sm_75;compute_80,sm_80;compute_86,sm_86;</CodeGeneration>
+      <AdditionalOptions>-Xcompiler "/wd 4819" %(AdditionalOptions)</AdditionalOptions>
+      <Include>./;../../Common</Include>
+      <Defines>WIN32</Defines>
+      <AdditionalOptions>--threads 0</AdditionalOptions>
+    </CudaCompile>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)'=='Debug'">
+    <ClCompile>
+      <Optimization>Disabled</Optimization>
+      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <LinkTimeCodeGeneration>Default</LinkTimeCodeGeneration>
+    </Link>
+    <CudaCompile>
+      <Runtime>MTd</Runtime>
+      <TargetMachinePlatform>64</TargetMachinePlatform>
+    </CudaCompile>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)'=='Release'">
+    <ClCompile>
+      <Optimization>MaxSpeed</Optimization>
+      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <GenerateDebugInformation>false</GenerateDebugInformation>
+      <LinkTimeCodeGeneration>UseLinkTimeCodeGeneration</LinkTimeCodeGeneration>
+    </Link>
+    <CudaCompile>
+      <Runtime>MT</Runtime>
+      <TargetMachinePlatform>64</TargetMachinePlatform>
+    </CudaCompile>
+  </ItemDefinitionGroup>
+  <ItemGroup>
+    <CudaCompile Include="BlackScholes.cu" />
+    <ClCompile Include="BlackScholes_gold.cpp" />
+    <None Include="BlackScholes_kernel.cuh" />
+  </ItemGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <ImportGroup Label="ExtensionTargets">
+    <Import Project="$(CUDAPropsPath)\CUDA 11.5.targets" />
+  </ImportGroup>
+</Project>

Samples/BlackScholes/Makefile

@@ -0,0 +1,365 @@
+################################################################################
+# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#  * Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#  * Neither the name of NVIDIA CORPORATION nor the names of its
+#    contributors may be used to endorse or promote products derived
+#    from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+################################################################################
+#
+# Makefile project only supported on Mac OS X and Linux Platforms)
+#
+################################################################################
+
+# Location of the CUDA Toolkit
+CUDA_PATH ?= /usr/local/cuda
+
+##############################
+# start deprecated interface #
+##############################
+ifeq ($(x86_64),1)
+    $(info WARNING - x86_64 variable has been deprecated)
+    $(info WARNING - please use TARGET_ARCH=x86_64 instead)
+    TARGET_ARCH ?= x86_64
+endif
+ifeq ($(ARMv7),1)
+    $(info WARNING - ARMv7 variable has been deprecated)
+    $(info WARNING - please use TARGET_ARCH=armv7l instead)
+    TARGET_ARCH ?= armv7l
+endif
+ifeq ($(aarch64),1)
+    $(info WARNING - aarch64 variable has been deprecated)
+    $(info WARNING - please use TARGET_ARCH=aarch64 instead)
+    TARGET_ARCH ?= aarch64
+endif
+ifeq ($(ppc64le),1)
+    $(info WARNING - ppc64le variable has been deprecated)
+    $(info WARNING - please use TARGET_ARCH=ppc64le instead)
+    TARGET_ARCH ?= ppc64le
+endif
+ifneq ($(GCC),)
+    $(info WARNING - GCC variable has been deprecated)
+    $(info WARNING - please use HOST_COMPILER=$(GCC) instead)
+    HOST_COMPILER ?= $(GCC)
+endif
+ifneq ($(abi),)
+    $(error ERROR - abi variable has been removed)
+endif
+############################
+# end deprecated interface #
+############################
+
+# architecture
+HOST_ARCH   := $(shell uname -m)
+TARGET_ARCH ?= $(HOST_ARCH)
+ifneq (,$(filter $(TARGET_ARCH),x86_64 aarch64 sbsa ppc64le armv7l))
+    ifneq ($(TARGET_ARCH),$(HOST_ARCH))
+        ifneq (,$(filter $(TARGET_ARCH),x86_64 aarch64 sbsa ppc64le))
+            TARGET_SIZE := 64
+        else ifneq (,$(filter $(TARGET_ARCH),armv7l))
+            TARGET_SIZE := 32
+        endif
+    else
+        TARGET_SIZE := $(shell getconf LONG_BIT)
+    endif
+else
+    $(error ERROR - unsupported value $(TARGET_ARCH) for TARGET_ARCH!)
+endif
+
+# sbsa and aarch64 systems look similar. Need to differentiate them at host level for now.
+ifeq ($(HOST_ARCH),aarch64)
+    ifeq ($(CUDA_PATH)/targets/sbsa-linux,$(shell ls -1d $(CUDA_PATH)/targets/sbsa-linux 2>/dev/null))
+        HOST_ARCH := sbsa
+        TARGET_ARCH := sbsa
+    endif
+endif
+
+ifneq ($(TARGET_ARCH),$(HOST_ARCH))
+    ifeq (,$(filter $(HOST_ARCH)-$(TARGET_ARCH),aarch64-armv7l x86_64-armv7l x86_64-aarch64 x86_64-sbsa x86_64-ppc64le))
+        $(error ERROR - cross compiling from $(HOST_ARCH) to $(TARGET_ARCH) is not supported!)
+    endif
+endif
+
+# When on native aarch64 system with userspace of 32-bit, change TARGET_ARCH to armv7l
+ifeq ($(HOST_ARCH)-$(TARGET_ARCH)-$(TARGET_SIZE),aarch64-aarch64-32)
+    TARGET_ARCH = armv7l
+endif
+
+# operating system
+HOST_OS   := $(shell uname -s 2>/dev/null | tr "[:upper:]" "[:lower:]")
+TARGET_OS ?= $(HOST_OS)
+ifeq (,$(filter $(TARGET_OS),linux darwin qnx android))
+    $(error ERROR - unsupported value $(TARGET_OS) for TARGET_OS!)
+endif
+
+# host compiler
+ifeq ($(TARGET_OS),darwin)
+    ifeq ($(shell expr `xcodebuild -version | grep -i xcode | awk '{print $$2}' | cut -d'.' -f1` \>= 5),1)
+        HOST_COMPILER ?= clang++
+    endif
+else ifneq ($(TARGET_ARCH),$(HOST_ARCH))
+    ifeq ($(HOST_ARCH)-$(TARGET_ARCH),x86_64-armv7l)
+        ifeq ($(TARGET_OS),linux)
+            HOST_COMPILER ?= arm-linux-gnueabihf-g++
+        else ifeq ($(TARGET_OS),qnx)
+            ifeq ($(QNX_HOST),)
+                $(error ERROR - QNX_HOST must be passed to the QNX host toolchain)
+            endif
+            ifeq ($(QNX_TARGET),)
+                $(error ERROR - QNX_TARGET must be passed to the QNX target toolchain)
+            endif
+            export QNX_HOST
+            export QNX_TARGET
+            HOST_COMPILER ?= $(QNX_HOST)/usr/bin/arm-unknown-nto-qnx6.6.0eabi-g++
+        else ifeq ($(TARGET_OS),android)
+            HOST_COMPILER ?= arm-linux-androideabi-g++
+        endif
+    else ifeq ($(TARGET_ARCH),aarch64)
+        ifeq ($(TARGET_OS), linux)
+            HOST_COMPILER ?= aarch64-linux-gnu-g++
+        else ifeq ($(TARGET_OS),qnx)
+            ifeq ($(QNX_HOST),)
+                $(error ERROR - QNX_HOST must be passed to the QNX host toolchain)
+            endif
+            ifeq ($(QNX_TARGET),)
+                $(error ERROR - QNX_TARGET must be passed to the QNX target toolchain)
+            endif
+            export QNX_HOST
+            export QNX_TARGET
+            HOST_COMPILER ?= $(QNX_HOST)/usr/bin/q++
+        else ifeq ($(TARGET_OS), android)
+            HOST_COMPILER ?= aarch64-linux-android-clang++
+        endif
+    else ifeq ($(TARGET_ARCH),sbsa)
+        HOST_COMPILER ?= aarch64-linux-gnu-g++
+    else ifeq ($(TARGET_ARCH),ppc64le)
+        HOST_COMPILER ?= powerpc64le-linux-gnu-g++
+    endif
+endif
+HOST_COMPILER ?= g++
+NVCC          := $(CUDA_PATH)/bin/nvcc -ccbin $(HOST_COMPILER)
+
+# internal flags
+NVCCFLAGS   := -m${TARGET_SIZE}
+CCFLAGS     :=
+LDFLAGS     :=
+
+# build flags
+ifeq ($(TARGET_OS),darwin)
+    LDFLAGS += -rpath $(CUDA_PATH)/lib
+    CCFLAGS += -arch $(HOST_ARCH)
+else ifeq ($(HOST_ARCH)-$(TARGET_ARCH)-$(TARGET_OS),x86_64-armv7l-linux)
+    LDFLAGS += --dynamic-linker=/lib/ld-linux-armhf.so.3
+    CCFLAGS += -mfloat-abi=hard
+else ifeq ($(TARGET_OS),android)
+    LDFLAGS += -pie
+    CCFLAGS += -fpie -fpic -fexceptions
+endif
+
+ifneq ($(TARGET_ARCH),$(HOST_ARCH))
+    ifeq ($(TARGET_ARCH)-$(TARGET_OS),armv7l-linux)
+        ifneq ($(TARGET_FS),)
+            GCCVERSIONLTEQ46 := $(shell expr `$(HOST_COMPILER) -dumpversion` \<= 4.6)
+            ifeq ($(GCCVERSIONLTEQ46),1)
+                CCFLAGS += --sysroot=$(TARGET_FS)
+            endif
+            LDFLAGS += --sysroot=$(TARGET_FS)
+            LDFLAGS += -rpath-link=$(TARGET_FS)/lib
+            LDFLAGS += -rpath-link=$(TARGET_FS)/usr/lib
+            LDFLAGS += -rpath-link=$(TARGET_FS)/usr/lib/arm-linux-gnueabihf
+        endif
+    endif
+    ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-linux)
+        ifneq ($(TARGET_FS),)
+            GCCVERSIONLTEQ46 := $(shell expr `$(HOST_COMPILER) -dumpversion` \<= 4.6)
+            ifeq ($(GCCVERSIONLTEQ46),1)
+                CCFLAGS += --sysroot=$(TARGET_FS)
+            endif
+            LDFLAGS += --sysroot=$(TARGET_FS)
+            LDFLAGS += -rpath-link=$(TARGET_FS)/lib -L$(TARGET_FS)/lib
+            LDFLAGS += -rpath-link=$(TARGET_FS)/lib/aarch64-linux-gnu -L$(TARGET_FS)/lib/aarch64-linux-gnu
+            LDFLAGS += -rpath-link=$(TARGET_FS)/usr/lib -L$(TARGET_FS)/usr/lib
+            LDFLAGS += -rpath-link=$(TARGET_FS)/usr/lib/aarch64-linux-gnu -L$(TARGET_FS)/usr/lib/aarch64-linux-gnu
+            LDFLAGS += --unresolved-symbols=ignore-in-shared-libs
+            CCFLAGS += -isystem=$(TARGET_FS)/usr/include -I$(TARGET_FS)/usr/include -I$(TARGET_FS)/usr/include/libdrm
+            CCFLAGS += -isystem=$(TARGET_FS)/usr/include/aarch64-linux-gnu -I$(TARGET_FS)/usr/include/aarch64-linux-gnu
+        endif
+    endif
+    ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-qnx)
+        NVCCFLAGS += --qpp-config 5.4.0,gcc_ntoaarch64le
+        CCFLAGS += -DWIN_INTERFACE_CUSTOM -I/usr/include/aarch64-qnx-gnu
+        LDFLAGS += -lsocket
+        LDFLAGS += -L/usr/lib/aarch64-qnx-gnu
+        CCFLAGS += "-Wl\,-rpath-link\,/usr/lib/aarch64-qnx-gnu"
+        ifdef TARGET_OVERRIDE
+            LDFLAGS += -lslog2
+        endif
+
+        ifneq ($(TARGET_FS),)
+            LDFLAGS += -L$(TARGET_FS)/usr/lib
+            CCFLAGS += "-Wl\,-rpath-link\,$(TARGET_FS)/usr/lib"
+            LDFLAGS += -L$(TARGET_FS)/usr/libnvidia
+            CCFLAGS += "-Wl\,-rpath-link\,$(TARGET_FS)/usr/libnvidia"
+            CCFLAGS += -I$(TARGET_FS)/../include
+        endif
+    endif
+endif
+
+ifdef TARGET_OVERRIDE # cuda toolkit targets override
+    NVCCFLAGS += -target-dir $(TARGET_OVERRIDE)
+endif
+
+# Install directory of different arch
+CUDA_INSTALL_TARGET_DIR :=
+ifeq ($(TARGET_ARCH)-$(TARGET_OS),armv7l-linux)
+    CUDA_INSTALL_TARGET_DIR = targets/armv7-linux-gnueabihf/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-linux)
+    CUDA_INSTALL_TARGET_DIR = targets/aarch64-linux/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),sbsa-linux)
+    CUDA_INSTALL_TARGET_DIR = targets/sbsa-linux/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),armv7l-android)
+    CUDA_INSTALL_TARGET_DIR = targets/armv7-linux-androideabi/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-android)
+    CUDA_INSTALL_TARGET_DIR = targets/aarch64-linux-androideabi/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),armv7l-qnx)
+    CUDA_INSTALL_TARGET_DIR = targets/ARMv7-linux-QNX/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-qnx)
+    CUDA_INSTALL_TARGET_DIR = targets/aarch64-qnx/
+else ifeq ($(TARGET_ARCH),ppc64le)
+    CUDA_INSTALL_TARGET_DIR = targets/ppc64le-linux/
+endif
+
+# Debug build flags
+ifeq ($(dbg),1)
+      NVCCFLAGS += -g -G
+      BUILD_TYPE := debug
+else
+      BUILD_TYPE := release
+endif
+
+ALL_CCFLAGS :=
+ALL_CCFLAGS += $(NVCCFLAGS)
+ALL_CCFLAGS += $(EXTRA_NVCCFLAGS)
+ALL_CCFLAGS += $(addprefix -Xcompiler ,$(CCFLAGS))
+ALL_CCFLAGS += $(addprefix -Xcompiler ,$(EXTRA_CCFLAGS))
+
+SAMPLE_ENABLED := 1
+
+ALL_LDFLAGS :=
+ALL_LDFLAGS += $(ALL_CCFLAGS)
+ALL_LDFLAGS += $(addprefix -Xlinker ,$(LDFLAGS))
+ALL_LDFLAGS += $(addprefix -Xlinker ,$(EXTRA_LDFLAGS))
+
+# Common includes and paths for CUDA
+INCLUDES  := -I../../Common
+LIBRARIES :=
+
+################################################################################
+
+#Detect if installed version of GCC supports required C++11
+ifeq ($(TARGET_OS),linux)
+    empty :=
+    space := $(empty) $(empty)
+    GCCVERSIONSTRING := $(shell expr `$(HOST_COMPILER) -dumpversion`)
+#Create version number without "."
+    GCCVERSION := $(shell expr `echo $(GCCVERSIONSTRING)` | cut -f1 -d.)
+    GCCVERSION += $(shell expr `echo $(GCCVERSIONSTRING)` | cut -f2 -d.)
+    GCCVERSION += $(shell expr `echo $(GCCVERSIONSTRING)` | cut -f3 -d.)
+# Make sure the version number has at least 3 decimals
+    GCCVERSION += 00
+# Remove spaces from the version number
+    GCCVERSION := $(subst $(space),$(empty),$(GCCVERSION))
+#$(warning $(GCCVERSION))
+
+    IS_MIN_VERSION := $(shell expr `echo $(GCCVERSION)` \>= 51000)
+
+    ifeq ($(IS_MIN_VERSION), 1)
+        $(info >>> GCC Version is greater or equal to 5.1.0 <<<)
+    else
+        $(info >>> Waiving build. Minimum GCC version required is 5.1.0<<<)
+        SAMPLE_ENABLED := 0
+    endif
+endif
+
+# Gencode arguments
+ifeq ($(TARGET_ARCH),$(filter $(TARGET_ARCH),armv7l aarch64))
+SMS ?= 35 37 50 52 60 61 70 72 75 80 86
+else
+SMS ?= 35 37 50 52 60 61 70 75 80 86
+endif
+
+ifeq ($(SMS),)
+$(info >>> WARNING - no SM architectures have been specified - waiving sample <<<)
+SAMPLE_ENABLED := 0
+endif
+
+ifeq ($(GENCODE_FLAGS),)
+# Generate SASS code for each SM architecture listed in $(SMS)
+$(foreach sm,$(SMS),$(eval GENCODE_FLAGS += -gencode arch=compute_$(sm),code=sm_$(sm)))
+
+# Generate PTX code from the highest SM architecture in $(SMS) to guarantee forward-compatibility
+HIGHEST_SM := $(lastword $(sort $(SMS)))
+ifneq ($(HIGHEST_SM),)
+GENCODE_FLAGS += -gencode arch=compute_$(HIGHEST_SM),code=compute_$(HIGHEST_SM)
+endif
+endif
+
+ALL_CCFLAGS += -maxrregcount=16 --threads 0 --std=c++11
+
+ifeq ($(SAMPLE_ENABLED),0)
+EXEC ?= @echo "[@]"
+endif
+
+################################################################################
+
+# Target rules
+all: build
+
+build: BlackScholes
+
+check.deps:
+ifeq ($(SAMPLE_ENABLED),0)
+	@echo "Sample will be waived due to the above missing dependencies"
+else
+	@echo "Sample is ready - all dependencies have been met"
+endif
+
+BlackScholes.o:BlackScholes.cu
+	$(EXEC) $(NVCC) $(INCLUDES) $(ALL_CCFLAGS) $(GENCODE_FLAGS) -o $@ -c $<
+
+BlackScholes_gold.o:BlackScholes_gold.cpp
+	$(EXEC) $(NVCC) $(INCLUDES) $(ALL_CCFLAGS) $(GENCODE_FLAGS) -o $@ -c $<
+
+BlackScholes: BlackScholes.o BlackScholes_gold.o
+	$(EXEC) $(NVCC) $(ALL_LDFLAGS) $(GENCODE_FLAGS) -o $@ $+ $(LIBRARIES)
+	$(EXEC) mkdir -p ../../bin/$(TARGET_ARCH)/$(TARGET_OS)/$(BUILD_TYPE)
+	$(EXEC) cp $@ ../../bin/$(TARGET_ARCH)/$(TARGET_OS)/$(BUILD_TYPE)
+
+run: build
+	$(EXEC) ./BlackScholes
+
+clean:
+	rm -f BlackScholes BlackScholes.o BlackScholes_gold.o
+	rm -rf ../../bin/$(TARGET_ARCH)/$(TARGET_OS)/$(BUILD_TYPE)/BlackScholes
+
+clobber: clean

Samples/BlackScholes/NsightEclipse.xml

@@ -0,0 +1,71 @@
+<?xml version="1.0" encoding="UTF-8"?> 
+<!DOCTYPE entry SYSTEM "SamplesInfo.dtd">
+<entry>
+  <name>BlackScholes</name>
+  <cflags>
+    <flag>-maxrregcount=16</flag>
+  </cflags>
+  <description><![CDATA[This sample evaluates fair call and put prices for a given set of European options by Black-Scholes formula.]]></description>
+  <devicecompilation>whole</devicecompilation>
+  <includepaths>
+    <path>./</path>
+    <path>../</path>
+    <path>../../Common</path>
+  </includepaths>
+  <keyconcepts>
+    <concept level="basic">Computational Finance</concept>
+  </keyconcepts>
+  <keywords>
+    <keyword>CUDA</keyword>
+    <keyword>Computational Finance</keyword>
+    <keyword>option pricing</keyword>
+    <keyword>Black-Scholes</keyword>
+  </keywords>
+  <libraries>
+  </libraries>
+  <librarypaths>
+  </librarypaths>
+  <nsight_eclipse>true</nsight_eclipse>
+  <primary_file>BlackScholes.cu</primary_file>
+  <scopes>
+    <scope>1:CUDA Basic Topics</scope>
+    <scope>3:Computational Finance</scope>
+  </scopes>
+  <sm-arch>sm35</sm-arch>
+  <sm-arch>sm37</sm-arch>
+  <sm-arch>sm50</sm-arch>
+  <sm-arch>sm52</sm-arch>
+  <sm-arch>sm60</sm-arch>
+  <sm-arch>sm61</sm-arch>
+  <sm-arch>sm70</sm-arch>
+  <sm-arch>sm72</sm-arch>
+  <sm-arch>sm75</sm-arch>
+  <sm-arch>sm80</sm-arch>
+  <sm-arch>sm86</sm-arch>
+  <supported_envs>
+    <env>
+      <arch>x86_64</arch>
+      <platform>linux</platform>
+    </env>
+    <env>
+      <platform>windows7</platform>
+    </env>
+    <env>
+      <arch>x86_64</arch>
+      <platform>macosx</platform>
+    </env>
+    <env>
+      <arch>arm</arch>
+    </env>
+    <env>
+      <arch>ppc64le</arch>
+      <platform>linux</platform>
+    </env>
+  </supported_envs>
+  <supported_sm_architectures>
+    <include>all</include>
+  </supported_sm_architectures>
+  <title>Black-Scholes Option Pricing</title>
+  <type>exe</type>
+  <whitepaper>doc\BlackScholes.pdf</whitepaper>
+</entry>

Samples/BlackScholes/README.md

@@ -0,0 +1,67 @@
+# BlackScholes - Black-Scholes Option Pricing
+
+## Description
+
+This sample evaluates fair call and put prices for a given set of European options by Black-Scholes formula.
+
+## Key Concepts
+
+Computational Finance
+
+## Supported SM Architectures
+
+[SM 3.5 ](https://developer.nvidia.com/cuda-gpus)  [SM 3.7 ](https://developer.nvidia.com/cuda-gpus)  [SM 5.0 ](https://developer.nvidia.com/cuda-gpus)  [SM 5.2 ](https://developer.nvidia.com/cuda-gpus)  [SM 6.0 ](https://developer.nvidia.com/cuda-gpus)  [SM 6.1 ](https://developer.nvidia.com/cuda-gpus)  [SM 7.0 ](https://developer.nvidia.com/cuda-gpus)  [SM 7.2 ](https://developer.nvidia.com/cuda-gpus)  [SM 7.5 ](https://developer.nvidia.com/cuda-gpus)  [SM 8.0 ](https://developer.nvidia.com/cuda-gpus)  [SM 8.6 ](https://developer.nvidia.com/cuda-gpus)
+
+## Supported OSes
+
+Linux, Windows
+
+## Supported CPU Architecture
+
+x86_64, ppc64le, armv7l
+
+## CUDA APIs involved
+
+## Prerequisites
+
+Download and install the [CUDA Toolkit 11.5](https://developer.nvidia.com/cuda-downloads) for your corresponding platform.
+
+## Build and Run
+
+### Windows
+The Windows samples are built using the Visual Studio IDE. Solution files (.sln) are provided for each supported version of Visual Studio, using the format:
+```
+*_vs<version>.sln - for Visual Studio <version>
+```
+Each individual sample has its own set of solution files in its directory:
+
+To build/examine all the samples at once, the complete solution files should be used. To build/examine a single sample, the individual sample solution files should be used.
+> **Note:** Some samples require that the Microsoft DirectX SDK (June 2010 or newer) be installed and that the VC++ directory paths are properly set up (**Tools > Options...**). Check DirectX Dependencies section for details."
+
+### Linux
+The Linux samples are built using makefiles. To use the makefiles, change the current directory to the sample directory you wish to build, and run make:
+```
+$ cd <sample_dir>
+$ make
+```
+The samples makefiles can take advantage of certain options:
+*  **TARGET_ARCH=<arch>** - cross-compile targeting a specific architecture. Allowed architectures are x86_64, ppc64le, armv7l.
+    By default, TARGET_ARCH is set to HOST_ARCH. On a x86_64 machine, not setting TARGET_ARCH is the equivalent of setting TARGET_ARCH=x86_64.<br/>
+`$ make TARGET_ARCH=x86_64` <br/> `$ make TARGET_ARCH=ppc64le` <br/> `$ make TARGET_ARCH=armv7l` <br/>
+    See [here](http://docs.nvidia.com/cuda/cuda-samples/index.html#cross-samples) for more details.
+*   **dbg=1** - build with debug symbols
+    ```
+    $ make dbg=1
+    ```
+*   **SMS="A B ..."** - override the SM architectures for which the sample will be built, where `"A B ..."` is a space-delimited list of SM architectures. For example, to generate SASS for SM 50 and SM 60, use `SMS="50 60"`.
+    ```
+    $ make SMS="50 60"
+    ```
+
+*  **HOST_COMPILER=<host_compiler>** - override the default g++ host compiler. See the [Linux Installation Guide](http://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html#system-requirements) for a list of supported host compilers.
+```
+    $ make HOST_COMPILER=g++
+```
+
+## References (for more details)
+

Samples/BlackScholes_nvrtc/.vscode/c_cpp_properties.json

@@ -0,0 +1,18 @@
+{
+    "configurations": [
+        {
+            "name": "Linux",
+            "includePath": [
+                "${workspaceFolder}/**",
+                "${workspaceFolder}/../../Common"
+            ],
+            "defines": [],
+            "compilerPath": "/usr/local/cuda/bin/nvcc",
+            "cStandard": "gnu17",
+            "cppStandard": "gnu++14",
+            "intelliSenseMode": "linux-gcc-x64",
+            "configurationProvider": "ms-vscode.makefile-tools"
+        }
+    ],
+    "version": 4
+}

Samples/BlackScholes_nvrtc/.vscode/extensions.json

@@ -0,0 +1,7 @@
+{
+    "recommendations": [
+        "nvidia.nsight-vscode-edition",
+        "ms-vscode.cpptools",
+        "ms-vscode.makefile-tools"
+    ]
+}

Samples/BlackScholes_nvrtc/.vscode/launch.json

@@ -0,0 +1,10 @@
+{
+    "configurations": [
+        {
+            "name": "CUDA C++: Launch",
+            "type": "cuda-gdb",
+            "request": "launch",
+            "program": "${workspaceFolder}/BlackScholes_nvrtc"
+        }
+    ]
+}

Samples/BlackScholes_nvrtc/.vscode/tasks.json

@@ -0,0 +1,15 @@
+{
+    "version": "2.0.0",
+    "tasks": [
+        {
+            "label": "sample",
+            "type": "shell",
+            "command": "make dbg=1",
+            "problemMatcher": ["$nvcc"],
+            "group": {
+                "kind": "build",
+                "isDefault": true
+            }
+        }
+    ]
+}

Samples/BlackScholes_nvrtc/BlackScholes.cpp

@@ -0,0 +1,269 @@
+/* Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * This sample evaluates fair call and put prices for a
+ * given set of European options by Black-Scholes formula.
+ * See supplied whitepaper for more explanations.
+ */
+
+#include <cuda_runtime.h>
+#include <nvrtc_helper.h>
+
+#include <helper_functions.h>  // helper functions for string parsing
+
+////////////////////////////////////////////////////////////////////////////////
+// Process an array of optN options on CPU
+////////////////////////////////////////////////////////////////////////////////
+
+extern "C" void BlackScholesCPU(float *h_CallResult, float *h_PutResult,
+                                float *h_StockPrice, float *h_OptionStrike,
+                                float *h_OptionYears, float Riskfree,
+                                float Volatility, int optN);
+
+////////////////////////////////////////////////////////////////////////////////
+// Process an array of OptN options on GPU
+////////////////////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////////////////////
+
+// Helper function, returning uniformly distributed
+// random float in [low, high] range
+////////////////////////////////////////////////////////////////////////////////
+
+float RandFloat(float low, float high) {
+  float t = (float)rand() / (float)RAND_MAX;
+  return (1.0f - t) * low + t * high;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// Data configuration
+////////////////////////////////////////////////////////////////////////////////
+
+const int OPT_N = 4000000;
+const int NUM_ITERATIONS = 512;
+const int OPT_SZ = OPT_N * sizeof(float);
+const float RISKFREE = 0.02f;
+const float VOLATILITY = 0.30f;
+
+#define DIV_UP(a, b) (((a) + (b)-1) / (b))
+
+////////////////////////////////////////////////////////////////////////////////
+// Main program
+////////////////////////////////////////////////////////////////////////////////
+
+int main(int argc, char **argv) {
+  // Start logs
+  printf("[%s] - Starting...\n", argv[0]);
+
+  //'h_' prefix - CPU (host) memory space
+  float
+      // Results calculated by CPU for reference
+      *h_CallResultCPU,
+      *h_PutResultCPU,
+      // CPU copy of GPU results
+      *h_CallResultGPU, *h_PutResultGPU,
+      // CPU instance of input data
+      *h_StockPrice, *h_OptionStrike, *h_OptionYears;
+
+  //'d_' prefix - GPU (device) memory space
+  CUdeviceptr
+      // Results calculated by GPU
+      d_CallResult,
+      d_PutResult,
+
+      // GPU instance of input data
+      d_StockPrice, d_OptionStrike, d_OptionYears;
+
+  double delta, ref, sum_delta, sum_ref, max_delta, L1norm, gpuTime;
+
+  StopWatchInterface *hTimer = NULL;
+  int i;
+
+  sdkCreateTimer(&hTimer);
+
+  printf("Initializing data...\n");
+  printf("...allocating CPU memory for options.\n");
+
+  h_CallResultCPU = (float *)malloc(OPT_SZ);
+  h_PutResultCPU = (float *)malloc(OPT_SZ);
+  h_CallResultGPU = (float *)malloc(OPT_SZ);
+  h_PutResultGPU = (float *)malloc(OPT_SZ);
+  h_StockPrice = (float *)malloc(OPT_SZ);
+  h_OptionStrike = (float *)malloc(OPT_SZ);
+  h_OptionYears = (float *)malloc(OPT_SZ);
+
+  char *cubin, *kernel_file;
+  size_t cubinSize;
+  kernel_file = sdkFindFilePath("BlackScholes_kernel.cuh", argv[0]);
+
+  // Compile the kernel BlackScholes_kernel.
+  compileFileToCUBIN(kernel_file, argc, argv, &cubin, &cubinSize, 0);
+  CUmodule module = loadCUBIN(cubin, argc, argv);
+
+  CUfunction kernel_addr;
+  checkCudaErrors(cuModuleGetFunction(&kernel_addr, module, "BlackScholesGPU"));
+
+  printf("...allocating GPU memory for options.\n");
+  checkCudaErrors(cuMemAlloc(&d_CallResult, OPT_SZ));
+  checkCudaErrors(cuMemAlloc(&d_PutResult, OPT_SZ));
+  checkCudaErrors(cuMemAlloc(&d_StockPrice, OPT_SZ));
+  checkCudaErrors(cuMemAlloc(&d_OptionStrike, OPT_SZ));
+  checkCudaErrors(cuMemAlloc(&d_OptionYears, OPT_SZ));
+
+  printf("...generating input data in CPU mem.\n");
+  srand(5347);
+
+  // Generate options set
+  for (i = 0; i < OPT_N; i++) {
+    h_CallResultCPU[i] = 0.0f;
+    h_PutResultCPU[i] = -1.0f;
+    h_StockPrice[i] = RandFloat(5.0f, 30.0f);
+    h_OptionStrike[i] = RandFloat(1.0f, 100.0f);
+    h_OptionYears[i] = RandFloat(0.25f, 10.0f);
+  }
+
+  printf("...copying input data to GPU mem.\n");
+  // Copy options data to GPU memory for further processing
+  checkCudaErrors(cuMemcpyHtoD(d_StockPrice, h_StockPrice, OPT_SZ));
+  checkCudaErrors(cuMemcpyHtoD(d_OptionStrike, h_OptionStrike, OPT_SZ));
+  checkCudaErrors(cuMemcpyHtoD(d_OptionYears, h_OptionYears, OPT_SZ));
+
+  printf("Data init done.\n\n");
+  printf("Executing Black-Scholes GPU kernel (%i iterations)...\n",
+         NUM_ITERATIONS);
+
+  sdkResetTimer(&hTimer);
+  sdkStartTimer(&hTimer);
+
+  dim3 cudaBlockSize(128, 1, 1);
+  dim3 cudaGridSize(DIV_UP(OPT_N / 2, 128), 1, 1);
+
+  float risk = RISKFREE;
+  float volatility = VOLATILITY;
+  int optval = OPT_N;
+
+  void *arr[] = {(void *)&d_CallResult,  (void *)&d_PutResult,
+                 (void *)&d_StockPrice,  (void *)&d_OptionStrike,
+                 (void *)&d_OptionYears, (void *)&risk,
+                 (void *)&volatility,    (void *)&optval};
+
+  for (i = 0; i < NUM_ITERATIONS; i++) {
+    checkCudaErrors(cuLaunchKernel(kernel_addr, cudaGridSize.x, cudaGridSize.y,
+                                   cudaGridSize.z, /* grid dim */
+                                   cudaBlockSize.x, cudaBlockSize.y,
+                                   cudaBlockSize.z, /* block dim */
+                                   0, 0,            /* shared mem, stream */
+                                   &arr[0],         /* arguments */
+                                   0));
+  }
+
+  checkCudaErrors(cuCtxSynchronize());
+
+  sdkStopTimer(&hTimer);
+  gpuTime = sdkGetTimerValue(&hTimer) / NUM_ITERATIONS;
+
+  // Both call and put is calculated
+  printf("Options count             : %i     \n", 2 * OPT_N);
+  printf("BlackScholesGPU() time    : %f msec\n", gpuTime);
+  printf("Effective memory bandwidth: %f GB/s\n",
+         ((double)(5 * OPT_N * sizeof(float)) * 1E-9) / (gpuTime * 1E-3));
+  printf("Gigaoptions per second    : %f     \n\n",
+         ((double)(2 * OPT_N) * 1E-9) / (gpuTime * 1E-3));
+  printf(
+      "BlackScholes, Throughput = %.4f GOptions/s, Time = %.5f s, Size = %u "
+      "options, NumDevsUsed = %u, Workgroup = %u\n",
+      (((double)(2.0 * OPT_N) * 1.0E-9) / (gpuTime * 1.0E-3)), gpuTime * 1e-3,
+      (2 * OPT_N), 1, 128);
+
+  printf("\nReading back GPU results...\n");
+
+  // Read back GPU results to compare them to CPU results
+  checkCudaErrors(cuMemcpyDtoH(h_CallResultGPU, d_CallResult, OPT_SZ));
+  checkCudaErrors(cuMemcpyDtoH(h_PutResultGPU, d_PutResult, OPT_SZ));
+
+  printf("Checking the results...\n");
+  printf("...running CPU calculations.\n\n");
+
+  // Calculate options values on CPU
+  BlackScholesCPU(h_CallResultCPU, h_PutResultCPU, h_StockPrice, h_OptionStrike,
+                  h_OptionYears, RISKFREE, VOLATILITY, OPT_N);
+
+  printf("Comparing the results...\n");
+  // Calculate max absolute difference and L1 distance
+  // between CPU and GPU results
+  sum_delta = 0;
+  sum_ref = 0;
+  max_delta = 0;
+
+  for (i = 0; i < OPT_N; i++) {
+    ref = h_CallResultCPU[i];
+    delta = fabs(h_CallResultCPU[i] - h_CallResultGPU[i]);
+
+    if (delta > max_delta) {
+      max_delta = delta;
+    }
+
+    sum_delta += delta;
+    sum_ref += fabs(ref);
+  }
+
+  L1norm = sum_delta / sum_ref;
+  printf("L1 norm: %E\n", L1norm);
+  printf("Max absolute error: %E\n\n", max_delta);
+
+  printf("Shutting down...\n");
+  printf("...releasing GPU memory.\n");
+
+  checkCudaErrors(cuMemFree(d_OptionYears));
+  checkCudaErrors(cuMemFree(d_OptionStrike));
+  checkCudaErrors(cuMemFree(d_StockPrice));
+  checkCudaErrors(cuMemFree(d_PutResult));
+  checkCudaErrors(cuMemFree(d_CallResult));
+
+  printf("...releasing CPU memory.\n");
+
+  free(h_OptionYears);
+  free(h_OptionStrike);
+  free(h_StockPrice);
+  free(h_PutResultGPU);
+  free(h_CallResultGPU);
+  free(h_PutResultCPU);
+  free(h_CallResultCPU);
+
+  sdkDeleteTimer(&hTimer);
+  printf("Shutdown done.\n");
+
+  printf("\n[%s] - Test Summary\n", argv[0]);
+
+  if (L1norm > 1e-6) {
+    printf("Test failed!\n");
+    exit(EXIT_FAILURE);
+  }
+
+  printf("Test passed\n");
+  exit(EXIT_SUCCESS);
+}

Samples/BlackScholes_nvrtc/BlackScholes_gold.cpp

@@ -0,0 +1,88 @@
+/* Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <math.h>
+
+///////////////////////////////////////////////////////////////////////////////
+// Polynomial approximation of cumulative normal distribution function
+///////////////////////////////////////////////////////////////////////////////
+
+static double CND(double d) {
+  const double A1 = 0.31938153;
+  const double A2 = -0.356563782;
+  const double A3 = 1.781477937;
+  const double A4 = -1.821255978;
+  const double A5 = 1.330274429;
+  const double RSQRT2PI = 0.39894228040143267793994605993438;
+
+  double K = 1.0 / (1.0 + 0.2316419 * fabs(d));
+
+  double cnd = RSQRT2PI * exp(-0.5 * d * d) *
+               (K * (A1 + K * (A2 + K * (A3 + K * (A4 + K * A5)))));
+
+  if (d > 0) cnd = 1.0 - cnd;
+
+  return cnd;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// Black-Scholes formula for both call and put
+///////////////////////////////////////////////////////////////////////////////
+static void BlackScholesBodyCPU(float &callResult, float &putResult,
+                                float Sf,  // Stock price
+                                float Xf,  // Option strike
+                                float Tf,  // Option years
+                                float Rf,  // Riskless rate
+                                float Vf  // Volatility rate
+                                ) {
+  double S = Sf, X = Xf, T = Tf, R = Rf, V = Vf;
+  double sqrtT = sqrt(T);
+  double d1 = (log(S / X) + (R + 0.5 * V * V) * T) / (V * sqrtT);
+  double d2 = d1 - V * sqrtT;
+  double CNDD1 = CND(d1);
+  double CNDD2 = CND(d2);
+
+  // Calculate Call and Put simultaneously
+  double expRT = exp(-R * T);
+
+  callResult = (float)(S * CNDD1 - X * expRT * CNDD2);
+  putResult = (float)(X * expRT * (1.0 - CNDD2) - S * (1.0 - CNDD1));
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// Process an array of optN options
+////////////////////////////////////////////////////////////////////////////////
+
+extern "C" void BlackScholesCPU(float *h_CallResult, float *h_PutResult,
+                                float *h_StockPrice, float *h_OptionStrike,
+                                float *h_OptionYears, float Riskfree,
+                                float Volatility, int optN) {
+  for (int opt = 0; opt < optN; opt++)
+    BlackScholesBodyCPU(h_CallResult[opt], h_PutResult[opt], h_StockPrice[opt],
+                        h_OptionStrike[opt], h_OptionYears[opt], Riskfree,
+                        Volatility);
+}

Samples/BlackScholes_nvrtc/BlackScholes_kernel.cuh

@@ -0,0 +1,103 @@
+/* Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+///////////////////////////////////////////////////////////////////////////////
+// Polynomial approximation of cumulative normal distribution function
+///////////////////////////////////////////////////////////////////////////////
+
+__device__ inline float cndGPU(float d) {
+  const float A1 = 0.31938153f;
+  const float A2 = -0.356563782f;
+  const float A3 = 1.781477937f;
+  const float A4 = -1.821255978f;
+  const float A5 = 1.330274429f;
+  const float RSQRT2PI = 0.39894228040143267793994605993438f;
+
+  float K = __fdividef(1.0f, (1.0f + 0.2316419f * fabsf(d)));
+
+  float cnd = RSQRT2PI * __expf(-0.5f * d * d) *
+              (K * (A1 + K * (A2 + K * (A3 + K * (A4 + K * A5)))));
+
+  if (d > 0) cnd = 1.0f - cnd;
+
+  return cnd;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// Black-Scholes formula for both call and put
+///////////////////////////////////////////////////////////////////////////////
+__device__ inline void BlackScholesBodyGPU(float &CallResult, float &PutResult,
+                                           float S,  // Stock price
+                                           float X,  // Option strike
+                                           float T,  // Option years
+                                           float R,  // Riskless rate
+                                           float V  // Volatility rate
+                                           ) {
+  float sqrtT, expRT;
+  float d1, d2, CNDD1, CNDD2;
+
+  sqrtT = __fdividef(1.0F, rsqrtf(T));
+  d1 = __fdividef(__logf(S / X) + (R + 0.5f * V * V) * T, V * sqrtT);
+  d2 = d1 - V * sqrtT;
+
+  CNDD1 = cndGPU(d1);
+  CNDD2 = cndGPU(d2);
+
+  // Calculate Call and Put simultaneously
+  expRT = __expf(-R * T);
+  CallResult = S * CNDD1 - X * expRT * CNDD2;
+  PutResult = X * expRT * (1.0f - CNDD2) - S * (1.0f - CNDD1);
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// Process an array of optN options on GPU
+////////////////////////////////////////////////////////////////////////////////
+extern "C" __launch_bounds__(128) __global__
+    void BlackScholesGPU(float2 *__restrict d_CallResult,
+                         float2 *__restrict d_PutResult,
+                         float2 *__restrict d_StockPrice,
+                         float2 *__restrict d_OptionStrike,
+                         float2 *__restrict d_OptionYears, float Riskfree,
+                         float Volatility, int optN) {
+  ////Thread index
+  const int opt = blockDim.x * blockIdx.x + threadIdx.x;
+
+  // Calculating 2 options per thread to increase ILP (instruction level
+  // parallelism)
+  if (opt < (optN / 2)) {
+    float callResult1, callResult2;
+    float putResult1, putResult2;
+    BlackScholesBodyGPU(callResult1, putResult1, d_StockPrice[opt].x,
+                        d_OptionStrike[opt].x, d_OptionYears[opt].x, Riskfree,
+                        Volatility);
+    BlackScholesBodyGPU(callResult2, putResult2, d_StockPrice[opt].y,
+                        d_OptionStrike[opt].y, d_OptionYears[opt].y, Riskfree,
+                        Volatility);
+    d_CallResult[opt] = make_float2(callResult1, callResult2);
+    d_PutResult[opt] = make_float2(putResult1, putResult2);
+  }
+}

Samples/BlackScholes_nvrtc/BlackScholes_nvrtc_vs2017.sln

@@ -0,0 +1,20 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio 2017
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "BlackScholes_nvrtc", "BlackScholes_nvrtc_vs2017.vcxproj", "{997E0757-EA74-4A4E-A0FC-47D8C8831A15}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|x64 = Debug|x64
+		Release|x64 = Release|x64
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Debug|x64.ActiveCfg = Debug|x64
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Debug|x64.Build.0 = Debug|x64
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Release|x64.ActiveCfg = Release|x64
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Release|x64.Build.0 = Release|x64
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+EndGlobal

Samples/BlackScholes_nvrtc/BlackScholes_nvrtc_vs2017.vcxproj

@@ -0,0 +1,114 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <CUDAPropsPath Condition="'$(CUDAPropsPath)'==''">$(VCTargetsPath)\BuildCustomizations</CUDAPropsPath>
+  </PropertyGroup>
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Debug|x64">
+      <Configuration>Debug</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|x64">
+      <Configuration>Release</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>{997E0757-EA74-4A4E-A0FC-47D8C8831A15}</ProjectGuid>
+    <RootNamespace>BlackScholes_nvrtc_vs2017</RootNamespace>
+    <ProjectName>BlackScholes_nvrtc</ProjectName>
+    <CudaToolkitCustomDir />
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(WindowsTargetPlatformVersion)'==''">
+    <LatestTargetPlatformVersion>$([Microsoft.Build.Utilities.ToolLocationHelper]::GetLatestSDKTargetPlatformVersion('Windows', '10.0'))</LatestTargetPlatformVersion>
+    <WindowsTargetPlatformVersion Condition="'$(WindowsTargetPlatformVersion)' == ''">$(LatestTargetPlatformVersion)</WindowsTargetPlatformVersion>
+    <TargetPlatformVersion>$(WindowsTargetPlatformVersion)</TargetPlatformVersion>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <PropertyGroup>
+    <ConfigurationType>Application</ConfigurationType>
+    <CharacterSet>MultiByte</CharacterSet>
+    <PlatformToolset>v141</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)'=='Debug'">
+    <UseDebugLibraries>true</UseDebugLibraries>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)'=='Release'">
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <ImportGroup Label="ExtensionSettings">
+    <Import Project="$(CUDAPropsPath)\CUDA 11.5.props" />
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets">
+    <Import Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" />
+  </ImportGroup>
+  <PropertyGroup Label="UserMacros" />
+  <PropertyGroup>
+    <IntDir>$(Platform)/$(Configuration)/</IntDir>
+    <IncludePath>$(IncludePath)</IncludePath>
+    <CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRules />
+    <CodeAnalysisRuleAssemblies />
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Platform)'=='x64'">
+    <OutDir>../../bin/win64/$(Configuration)/</OutDir>
+  </PropertyGroup>
+  <ItemDefinitionGroup>
+    <ClCompile>
+      <WarningLevel>Level3</WarningLevel>
+      <PreprocessorDefinitions>WIN32;_MBCS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <AdditionalIncludeDirectories>./;$(CudaToolkitDir)/include;../../Common;$(CudaToolkitIncludeDir);$(CUDA_PATH)/include;</AdditionalIncludeDirectories>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <AdditionalDependencies>cuda.lib;nvrtc.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalLibraryDirectories>$(CudaToolkitLibDir);</AdditionalLibraryDirectories>
+      <OutputFile>$(OutDir)/BlackScholes_nvrtc.exe</OutputFile>
+    </Link>
+    <CudaCompile>
+      <CodeGeneration></CodeGeneration>
+      <AdditionalOptions>-Xcompiler "/wd 4819" %(AdditionalOptions)</AdditionalOptions>
+      <Include>./;../../Common</Include>
+      <Defines>WIN32</Defines>
+      <AdditionalOptions>--threads 0</AdditionalOptions>
+    </CudaCompile>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)'=='Debug'">
+    <ClCompile>
+      <Optimization>Disabled</Optimization>
+      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <LinkTimeCodeGeneration>Default</LinkTimeCodeGeneration>
+    </Link>
+    <CudaCompile>
+      <Runtime>MTd</Runtime>
+      <TargetMachinePlatform>64</TargetMachinePlatform>
+    </CudaCompile>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)'=='Release'">
+    <ClCompile>
+      <Optimization>MaxSpeed</Optimization>
+      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <GenerateDebugInformation>false</GenerateDebugInformation>
+      <LinkTimeCodeGeneration>UseLinkTimeCodeGeneration</LinkTimeCodeGeneration>
+    </Link>
+    <CudaCompile>
+      <Runtime>MT</Runtime>
+      <TargetMachinePlatform>64</TargetMachinePlatform>
+    </CudaCompile>
+  </ItemDefinitionGroup>
+  <ItemGroup>
+    <ClCompile Include="BlackScholes.cpp" />
+    <ClCompile Include="BlackScholes_gold.cpp" />
+    <None Include="BlackScholes_kernel.cuh" />
+  </ItemGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <ImportGroup Label="ExtensionTargets">
+    <Import Project="$(CUDAPropsPath)\CUDA 11.5.targets" />
+  </ImportGroup>
+</Project>

Samples/BlackScholes_nvrtc/BlackScholes_nvrtc_vs2019.sln

@@ -0,0 +1,20 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio 2019
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "BlackScholes_nvrtc", "BlackScholes_nvrtc_vs2019.vcxproj", "{997E0757-EA74-4A4E-A0FC-47D8C8831A15}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|x64 = Debug|x64
+		Release|x64 = Release|x64
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Debug|x64.ActiveCfg = Debug|x64
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Debug|x64.Build.0 = Debug|x64
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Release|x64.ActiveCfg = Release|x64
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Release|x64.Build.0 = Release|x64
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+EndGlobal

Samples/BlackScholes_nvrtc/BlackScholes_nvrtc_vs2019.vcxproj

@@ -0,0 +1,110 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <CUDAPropsPath Condition="'$(CUDAPropsPath)'==''">$(VCTargetsPath)\BuildCustomizations</CUDAPropsPath>
+  </PropertyGroup>
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Debug|x64">
+      <Configuration>Debug</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|x64">
+      <Configuration>Release</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>{997E0757-EA74-4A4E-A0FC-47D8C8831A15}</ProjectGuid>
+    <RootNamespace>BlackScholes_nvrtc_vs2019</RootNamespace>
+    <ProjectName>BlackScholes_nvrtc</ProjectName>
+    <CudaToolkitCustomDir />
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <PropertyGroup>
+    <ConfigurationType>Application</ConfigurationType>
+    <CharacterSet>MultiByte</CharacterSet>
+    <PlatformToolset>v142</PlatformToolset>
+    <WindowsTargetPlatformVersion>10.0</WindowsTargetPlatformVersion>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)'=='Debug'">
+    <UseDebugLibraries>true</UseDebugLibraries>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)'=='Release'">
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <ImportGroup Label="ExtensionSettings">
+    <Import Project="$(CUDAPropsPath)\CUDA 11.5.props" />
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets">
+    <Import Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" />
+  </ImportGroup>
+  <PropertyGroup Label="UserMacros" />
+  <PropertyGroup>
+    <IntDir>$(Platform)/$(Configuration)/</IntDir>
+    <IncludePath>$(IncludePath)</IncludePath>
+    <CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRules />
+    <CodeAnalysisRuleAssemblies />
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Platform)'=='x64'">
+    <OutDir>../../bin/win64/$(Configuration)/</OutDir>
+  </PropertyGroup>
+  <ItemDefinitionGroup>
+    <ClCompile>
+      <WarningLevel>Level3</WarningLevel>
+      <PreprocessorDefinitions>WIN32;_MBCS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <AdditionalIncludeDirectories>./;$(CudaToolkitDir)/include;../../Common;$(CudaToolkitIncludeDir);$(CUDA_PATH)/include;</AdditionalIncludeDirectories>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <AdditionalDependencies>cuda.lib;nvrtc.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalLibraryDirectories>$(CudaToolkitLibDir);</AdditionalLibraryDirectories>
+      <OutputFile>$(OutDir)/BlackScholes_nvrtc.exe</OutputFile>
+    </Link>
+    <CudaCompile>
+      <CodeGeneration></CodeGeneration>
+      <AdditionalOptions>-Xcompiler "/wd 4819" %(AdditionalOptions)</AdditionalOptions>
+      <Include>./;../../Common</Include>
+      <Defines>WIN32</Defines>
+      <AdditionalOptions>--threads 0</AdditionalOptions>
+    </CudaCompile>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)'=='Debug'">
+    <ClCompile>
+      <Optimization>Disabled</Optimization>
+      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <LinkTimeCodeGeneration>Default</LinkTimeCodeGeneration>
+    </Link>
+    <CudaCompile>
+      <Runtime>MTd</Runtime>
+      <TargetMachinePlatform>64</TargetMachinePlatform>
+    </CudaCompile>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)'=='Release'">
+    <ClCompile>
+      <Optimization>MaxSpeed</Optimization>
+      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <GenerateDebugInformation>false</GenerateDebugInformation>
+      <LinkTimeCodeGeneration>UseLinkTimeCodeGeneration</LinkTimeCodeGeneration>
+    </Link>
+    <CudaCompile>
+      <Runtime>MT</Runtime>
+      <TargetMachinePlatform>64</TargetMachinePlatform>
+    </CudaCompile>
+  </ItemDefinitionGroup>
+  <ItemGroup>
+    <ClCompile Include="BlackScholes.cpp" />
+    <ClCompile Include="BlackScholes_gold.cpp" />
+    <None Include="BlackScholes_kernel.cuh" />
+  </ItemGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <ImportGroup Label="ExtensionTargets">
+    <Import Project="$(CUDAPropsPath)\CUDA 11.5.targets" />
+  </ImportGroup>
+</Project>

Samples/BlackScholes_nvrtc/Makefile

@@ -0,0 +1,422 @@
+################################################################################
+# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#  * Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#  * Neither the name of NVIDIA CORPORATION nor the names of its
+#    contributors may be used to endorse or promote products derived
+#    from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+################################################################################
+#
+# Makefile project only supported on Mac OS X and Linux Platforms)
+#
+################################################################################
+
+# Location of the CUDA Toolkit
+CUDA_PATH ?= /usr/local/cuda
+
+##############################
+# start deprecated interface #
+##############################
+ifeq ($(x86_64),1)
+    $(info WARNING - x86_64 variable has been deprecated)
+    $(info WARNING - please use TARGET_ARCH=x86_64 instead)
+    TARGET_ARCH ?= x86_64
+endif
+ifeq ($(ARMv7),1)
+    $(info WARNING - ARMv7 variable has been deprecated)
+    $(info WARNING - please use TARGET_ARCH=armv7l instead)
+    TARGET_ARCH ?= armv7l
+endif
+ifeq ($(aarch64),1)
+    $(info WARNING - aarch64 variable has been deprecated)
+    $(info WARNING - please use TARGET_ARCH=aarch64 instead)
+    TARGET_ARCH ?= aarch64
+endif
+ifeq ($(ppc64le),1)
+    $(info WARNING - ppc64le variable has been deprecated)
+    $(info WARNING - please use TARGET_ARCH=ppc64le instead)
+    TARGET_ARCH ?= ppc64le
+endif
+ifneq ($(GCC),)
+    $(info WARNING - GCC variable has been deprecated)
+    $(info WARNING - please use HOST_COMPILER=$(GCC) instead)
+    HOST_COMPILER ?= $(GCC)
+endif
+ifneq ($(abi),)
+    $(error ERROR - abi variable has been removed)
+endif
+############################
+# end deprecated interface #
+############################
+
+# architecture
+HOST_ARCH   := $(shell uname -m)
+TARGET_ARCH ?= $(HOST_ARCH)
+ifneq (,$(filter $(TARGET_ARCH),x86_64 aarch64 sbsa ppc64le armv7l))
+    ifneq ($(TARGET_ARCH),$(HOST_ARCH))
+        ifneq (,$(filter $(TARGET_ARCH),x86_64 aarch64 sbsa ppc64le))
+            TARGET_SIZE := 64
+        else ifneq (,$(filter $(TARGET_ARCH),armv7l))
+            TARGET_SIZE := 32
+        endif
+    else
+        TARGET_SIZE := $(shell getconf LONG_BIT)
+    endif
+else
+    $(error ERROR - unsupported value $(TARGET_ARCH) for TARGET_ARCH!)
+endif
+
+# sbsa and aarch64 systems look similar. Need to differentiate them at host level for now.
+ifeq ($(HOST_ARCH),aarch64)
+    ifeq ($(CUDA_PATH)/targets/sbsa-linux,$(shell ls -1d $(CUDA_PATH)/targets/sbsa-linux 2>/dev/null))
+        HOST_ARCH := sbsa
+        TARGET_ARCH := sbsa
+    endif
+endif
+
+ifneq ($(TARGET_ARCH),$(HOST_ARCH))
+    ifeq (,$(filter $(HOST_ARCH)-$(TARGET_ARCH),aarch64-armv7l x86_64-armv7l x86_64-aarch64 x86_64-sbsa x86_64-ppc64le))
+        $(error ERROR - cross compiling from $(HOST_ARCH) to $(TARGET_ARCH) is not supported!)
+    endif
+endif
+
+# When on native aarch64 system with userspace of 32-bit, change TARGET_ARCH to armv7l
+ifeq ($(HOST_ARCH)-$(TARGET_ARCH)-$(TARGET_SIZE),aarch64-aarch64-32)
+    TARGET_ARCH = armv7l
+endif
+
+# operating system
+HOST_OS   := $(shell uname -s 2>/dev/null | tr "[:upper:]" "[:lower:]")
+TARGET_OS ?= $(HOST_OS)
+ifeq (,$(filter $(TARGET_OS),linux darwin qnx android))
+    $(error ERROR - unsupported value $(TARGET_OS) for TARGET_OS!)
+endif
+
+# host compiler
+ifeq ($(TARGET_OS),darwin)
+    ifeq ($(shell expr `xcodebuild -version | grep -i xcode | awk '{print $$2}' | cut -d'.' -f1` \>= 5),1)
+        HOST_COMPILER ?= clang++
+    endif
+else ifneq ($(TARGET_ARCH),$(HOST_ARCH))
+    ifeq ($(HOST_ARCH)-$(TARGET_ARCH),x86_64-armv7l)
+        ifeq ($(TARGET_OS),linux)
+            HOST_COMPILER ?= arm-linux-gnueabihf-g++
+        else ifeq ($(TARGET_OS),qnx)
+            ifeq ($(QNX_HOST),)
+                $(error ERROR - QNX_HOST must be passed to the QNX host toolchain)
+            endif
+            ifeq ($(QNX_TARGET),)
+                $(error ERROR - QNX_TARGET must be passed to the QNX target toolchain)
+            endif
+            export QNX_HOST
+            export QNX_TARGET
+            HOST_COMPILER ?= $(QNX_HOST)/usr/bin/arm-unknown-nto-qnx6.6.0eabi-g++
+        else ifeq ($(TARGET_OS),android)
+            HOST_COMPILER ?= arm-linux-androideabi-g++
+        endif
+    else ifeq ($(TARGET_ARCH),aarch64)
+        ifeq ($(TARGET_OS), linux)
+            HOST_COMPILER ?= aarch64-linux-gnu-g++
+        else ifeq ($(TARGET_OS),qnx)
+            ifeq ($(QNX_HOST),)
+                $(error ERROR - QNX_HOST must be passed to the QNX host toolchain)
+            endif
+            ifeq ($(QNX_TARGET),)
+                $(error ERROR - QNX_TARGET must be passed to the QNX target toolchain)
+            endif
+            export QNX_HOST
+            export QNX_TARGET
+            HOST_COMPILER ?= $(QNX_HOST)/usr/bin/q++
+        else ifeq ($(TARGET_OS), android)
+            HOST_COMPILER ?= aarch64-linux-android-clang++
+        endif
+    else ifeq ($(TARGET_ARCH),sbsa)
+        HOST_COMPILER ?= aarch64-linux-gnu-g++
+    else ifeq ($(TARGET_ARCH),ppc64le)
+        HOST_COMPILER ?= powerpc64le-linux-gnu-g++
+    endif
+endif
+HOST_COMPILER ?= g++
+NVCC          := $(CUDA_PATH)/bin/nvcc -ccbin $(HOST_COMPILER)
+
+# internal flags
+NVCCFLAGS   := -m${TARGET_SIZE}
+CCFLAGS     :=
+LDFLAGS     :=
+
+# build flags
+ifeq ($(TARGET_OS),darwin)
+    LDFLAGS += -rpath $(CUDA_PATH)/lib
+    CCFLAGS += -arch $(HOST_ARCH)
+else ifeq ($(HOST_ARCH)-$(TARGET_ARCH)-$(TARGET_OS),x86_64-armv7l-linux)
+    LDFLAGS += --dynamic-linker=/lib/ld-linux-armhf.so.3
+    CCFLAGS += -mfloat-abi=hard
+else ifeq ($(TARGET_OS),android)
+    LDFLAGS += -pie
+    CCFLAGS += -fpie -fpic -fexceptions
+endif
+
+ifneq ($(TARGET_ARCH),$(HOST_ARCH))
+    ifeq ($(TARGET_ARCH)-$(TARGET_OS),armv7l-linux)
+        ifneq ($(TARGET_FS),)
+            GCCVERSIONLTEQ46 := $(shell expr `$(HOST_COMPILER) -dumpversion` \<= 4.6)
+            ifeq ($(GCCVERSIONLTEQ46),1)
+                CCFLAGS += --sysroot=$(TARGET_FS)
+            endif
+            LDFLAGS += --sysroot=$(TARGET_FS)
+            LDFLAGS += -rpath-link=$(TARGET_FS)/lib
+            LDFLAGS += -rpath-link=$(TARGET_FS)/usr/lib
+            LDFLAGS += -rpath-link=$(TARGET_FS)/usr/lib/arm-linux-gnueabihf
+        endif
+    endif
+    ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-linux)
+        ifneq ($(TARGET_FS),)
+            GCCVERSIONLTEQ46 := $(shell expr `$(HOST_COMPILER) -dumpversion` \<= 4.6)
+            ifeq ($(GCCVERSIONLTEQ46),1)
+                CCFLAGS += --sysroot=$(TARGET_FS)
+            endif
+            LDFLAGS += --sysroot=$(TARGET_FS)
+            LDFLAGS += -rpath-link=$(TARGET_FS)/lib -L$(TARGET_FS)/lib
+            LDFLAGS += -rpath-link=$(TARGET_FS)/lib/aarch64-linux-gnu -L$(TARGET_FS)/lib/aarch64-linux-gnu
+            LDFLAGS += -rpath-link=$(TARGET_FS)/usr/lib -L$(TARGET_FS)/usr/lib
+            LDFLAGS += -rpath-link=$(TARGET_FS)/usr/lib/aarch64-linux-gnu -L$(TARGET_FS)/usr/lib/aarch64-linux-gnu
+            LDFLAGS += --unresolved-symbols=ignore-in-shared-libs
+            CCFLAGS += -isystem=$(TARGET_FS)/usr/include -I$(TARGET_FS)/usr/include -I$(TARGET_FS)/usr/include/libdrm
+            CCFLAGS += -isystem=$(TARGET_FS)/usr/include/aarch64-linux-gnu -I$(TARGET_FS)/usr/include/aarch64-linux-gnu
+        endif
+    endif
+    ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-qnx)
+        NVCCFLAGS += --qpp-config 5.4.0,gcc_ntoaarch64le
+        CCFLAGS += -DWIN_INTERFACE_CUSTOM -I/usr/include/aarch64-qnx-gnu
+        LDFLAGS += -lsocket
+        LDFLAGS += -L/usr/lib/aarch64-qnx-gnu
+        CCFLAGS += "-Wl\,-rpath-link\,/usr/lib/aarch64-qnx-gnu"
+        ifdef TARGET_OVERRIDE
+            LDFLAGS += -lslog2
+        endif
+
+        ifneq ($(TARGET_FS),)
+            LDFLAGS += -L$(TARGET_FS)/usr/lib
+            CCFLAGS += "-Wl\,-rpath-link\,$(TARGET_FS)/usr/lib"
+            LDFLAGS += -L$(TARGET_FS)/usr/libnvidia
+            CCFLAGS += "-Wl\,-rpath-link\,$(TARGET_FS)/usr/libnvidia"
+            CCFLAGS += -I$(TARGET_FS)/../include
+        endif
+    endif
+endif
+
+ifdef TARGET_OVERRIDE # cuda toolkit targets override
+    NVCCFLAGS += -target-dir $(TARGET_OVERRIDE)
+endif
+
+# Install directory of different arch
+CUDA_INSTALL_TARGET_DIR :=
+ifeq ($(TARGET_ARCH)-$(TARGET_OS),armv7l-linux)
+    CUDA_INSTALL_TARGET_DIR = targets/armv7-linux-gnueabihf/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-linux)
+    CUDA_INSTALL_TARGET_DIR = targets/aarch64-linux/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),sbsa-linux)
+    CUDA_INSTALL_TARGET_DIR = targets/sbsa-linux/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),armv7l-android)
+    CUDA_INSTALL_TARGET_DIR = targets/armv7-linux-androideabi/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-android)
+    CUDA_INSTALL_TARGET_DIR = targets/aarch64-linux-androideabi/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),armv7l-qnx)
+    CUDA_INSTALL_TARGET_DIR = targets/ARMv7-linux-QNX/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-qnx)
+    CUDA_INSTALL_TARGET_DIR = targets/aarch64-qnx/
+else ifeq ($(TARGET_ARCH),ppc64le)
+    CUDA_INSTALL_TARGET_DIR = targets/ppc64le-linux/
+endif
+
+# Debug build flags
+ifeq ($(dbg),1)
+      CCFLAGS += -g
+      BUILD_TYPE := debug
+else
+      BUILD_TYPE := release
+endif
+
+ALL_CCFLAGS :=
+ALL_CCFLAGS += $(NVCCFLAGS)
+ALL_CCFLAGS += $(EXTRA_NVCCFLAGS)
+ALL_CCFLAGS += $(addprefix -Xcompiler ,$(CCFLAGS))
+ALL_CCFLAGS += $(addprefix -Xcompiler ,$(EXTRA_CCFLAGS))
+
+UBUNTU = $(shell lsb_release -i -s 2>/dev/null | grep -i ubuntu)
+
+SAMPLE_ENABLED := 1
+
+# This sample is not supported on ARMv7
+ifeq ($(TARGET_ARCH),armv7l)
+  $(info >>> WARNING - BlackScholes_nvrtc is not supported on ARMv7 - waiving sample <<<)
+  SAMPLE_ENABLED := 0
+endif
+
+# This sample is not supported on QNX
+ifeq ($(TARGET_OS),qnx)
+  $(info >>> WARNING - BlackScholes_nvrtc is not supported on QNX - waiving sample <<<)
+  SAMPLE_ENABLED := 0
+endif
+
+ALL_LDFLAGS :=
+ALL_LDFLAGS += $(ALL_CCFLAGS)
+ALL_LDFLAGS += $(addprefix -Xlinker ,$(LDFLAGS))
+ALL_LDFLAGS += $(addprefix -Xlinker ,$(EXTRA_LDFLAGS))
+
+# Common includes and paths for CUDA
+INCLUDES  := -I../../Common
+LIBRARIES :=
+
+################################################################################
+
+# libNVRTC specific libraries
+ifeq ($(TARGET_OS),darwin)
+ LDFLAGS += -L$(CUDA_PATH)/lib -F/Library/Frameworks -framework CUDA
+endif
+
+#Detect if installed version of GCC supports required C++11
+ifeq ($(TARGET_OS),linux)
+    empty :=
+    space := $(empty) $(empty)
+    GCCVERSIONSTRING := $(shell expr `$(HOST_COMPILER) -dumpversion`)
+#Create version number without "."
+    GCCVERSION := $(shell expr `echo $(GCCVERSIONSTRING)` | cut -f1 -d.)
+    GCCVERSION += $(shell expr `echo $(GCCVERSIONSTRING)` | cut -f2 -d.)
+    GCCVERSION += $(shell expr `echo $(GCCVERSIONSTRING)` | cut -f3 -d.)
+# Make sure the version number has at least 3 decimals
+    GCCVERSION += 00
+# Remove spaces from the version number
+    GCCVERSION := $(subst $(space),$(empty),$(GCCVERSION))
+#$(warning $(GCCVERSION))
+
+    IS_MIN_VERSION := $(shell expr `echo $(GCCVERSION)` \>= 51000)
+
+    ifeq ($(IS_MIN_VERSION), 1)
+        $(info >>> GCC Version is greater or equal to 5.1.0 <<<)
+    else
+        $(info >>> Waiving build. Minimum GCC version required is 5.1.0<<<)
+        SAMPLE_ENABLED := 0
+    endif
+endif
+
+ifeq ($(TARGET_OS),darwin)
+  ALL_LDFLAGS += -Xcompiler -F/Library/Frameworks -Xlinker -framework -Xlinker CUDA
+else
+  ifeq ($(TARGET_ARCH),x86_64)
+    CUDA_SEARCH_PATH ?= $(CUDA_PATH)/lib64/stubs
+    CUDA_SEARCH_PATH += $(CUDA_PATH)/targets/x86_64-linux/lib/stubs
+  endif
+
+  ifeq ($(TARGET_ARCH)-$(TARGET_OS),armv7l-linux)
+    CUDA_SEARCH_PATH ?= $(CUDA_PATH)/targets/armv7-linux-gnueabihf/lib/stubs
+  endif
+
+  ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-linux)
+    CUDA_SEARCH_PATH ?= $(CUDA_PATH)/targets/aarch64-linux/lib/stubs
+  endif
+
+  ifeq ($(TARGET_ARCH)-$(TARGET_OS),sbsa-linux)
+    CUDA_SEARCH_PATH ?= $(CUDA_PATH)/targets/sbsa-linux/lib/stubs
+  endif
+
+  ifeq ($(TARGET_ARCH)-$(TARGET_OS),armv7l-android)
+    CUDA_SEARCH_PATH ?= $(CUDA_PATH)/targets/armv7-linux-androideabi/lib/stubs
+  endif
+
+  ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-android)
+    CUDA_SEARCH_PATH ?= $(CUDA_PATH)/targets/aarch64-linux-androideabi/lib/stubs
+  endif
+
+  ifeq ($(TARGET_ARCH)-$(TARGET_OS),armv7l-qnx)
+    CUDA_SEARCH_PATH ?= $(CUDA_PATH)/targets/ARMv7-linux-QNX/lib/stubs
+  endif
+
+  ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-qnx)
+    CUDA_SEARCH_PATH ?= $(CUDA_PATH)/targets/aarch64-qnx/lib/stubs
+    ifdef TARGET_OVERRIDE
+        CUDA_SEARCH_PATH := $(CUDA_PATH)/targets/$(TARGET_OVERRIDE)/lib/stubs
+    endif
+  endif
+
+  ifeq ($(TARGET_ARCH),ppc64le)
+    CUDA_SEARCH_PATH ?= $(CUDA_PATH)/targets/ppc64le-linux/lib/stubs
+  endif
+
+  ifeq ($(HOST_ARCH),ppc64le)
+    CUDA_SEARCH_PATH += $(CUDA_PATH)/lib64/stubs
+  endif
+
+  CUDALIB ?= $(shell find -L $(CUDA_SEARCH_PATH) -maxdepth 1 -name libcuda.so 2> /dev/null)
+  ifeq ("$(CUDALIB)","")
+    $(info >>> WARNING - libcuda.so not found, CUDA Driver is not installed.  Please re-install the driver. <<<)
+    SAMPLE_ENABLED := 0
+  else
+    CUDALIB := $(shell echo $(CUDALIB) | sed "s/ .*//" | sed "s/\/libcuda.so//" )
+    LIBRARIES += -L$(CUDALIB) -lcuda
+  endif
+endif
+
+ALL_CCFLAGS += --threads 0 --std=c++11
+
+INCLUDES += -I$(CUDA_PATH)/include
+
+LIBRARIES += -lnvrtc
+
+ifeq ($(SAMPLE_ENABLED),0)
+EXEC ?= @echo "[@]"
+endif
+
+################################################################################
+
+# Target rules
+all: build
+
+build: BlackScholes_nvrtc
+
+check.deps:
+ifeq ($(SAMPLE_ENABLED),0)
+	@echo "Sample will be waived due to the above missing dependencies"
+else
+	@echo "Sample is ready - all dependencies have been met"
+endif
+
+BlackScholes.o:BlackScholes.cpp
+	$(EXEC) $(HOST_COMPILER) $(INCLUDES) $(CCFLAGS) $(EXTRA_CCFLAGS) -o $@ -c $<
+
+BlackScholes_gold.o:BlackScholes_gold.cpp
+	$(EXEC) $(HOST_COMPILER) $(INCLUDES) $(CCFLAGS) $(EXTRA_CCFLAGS) -o $@ -c $<
+
+BlackScholes_nvrtc: BlackScholes.o BlackScholes_gold.o
+	$(EXEC) $(HOST_COMPILER) $(LDFLAGS) -o $@ $+ $(LIBRARIES)
+	$(EXEC) mkdir -p ../../bin/$(TARGET_ARCH)/$(TARGET_OS)/$(BUILD_TYPE)
+	$(EXEC) cp $@ ../../bin/$(TARGET_ARCH)/$(TARGET_OS)/$(BUILD_TYPE)
+
+run: build
+	$(EXEC) ./BlackScholes_nvrtc
+
+clean:
+	rm -f BlackScholes_nvrtc BlackScholes.o BlackScholes_gold.o
+	rm -rf ../../bin/$(TARGET_ARCH)/$(TARGET_OS)/$(BUILD_TYPE)/BlackScholes_nvrtc
+
+clobber: clean

Samples/BlackScholes_nvrtc/README.md

@@ -0,0 +1,71 @@
+# BlackScholes_nvrtc - Black-Scholes Option Pricing with libNVRTC
+
+## Description
+
+This sample evaluates fair call and put prices for a given set of European options by Black-Scholes formula, compiling the CUDA kernels involved at runtime using NVRTC.
+
+## Key Concepts
+
+Computational Finance, Runtime Compilation
+
+## Supported SM Architectures
+
+[SM 3.5 ](https://developer.nvidia.com/cuda-gpus)  [SM 3.7 ](https://developer.nvidia.com/cuda-gpus)  [SM 5.0 ](https://developer.nvidia.com/cuda-gpus)  [SM 5.2 ](https://developer.nvidia.com/cuda-gpus)  [SM 6.0 ](https://developer.nvidia.com/cuda-gpus)  [SM 6.1 ](https://developer.nvidia.com/cuda-gpus)  [SM 7.0 ](https://developer.nvidia.com/cuda-gpus)  [SM 7.2 ](https://developer.nvidia.com/cuda-gpus)  [SM 7.5 ](https://developer.nvidia.com/cuda-gpus)  [SM 8.0 ](https://developer.nvidia.com/cuda-gpus)  [SM 8.6 ](https://developer.nvidia.com/cuda-gpus)
+
+## Supported OSes
+
+Linux, Windows
+
+## Supported CPU Architecture
+
+x86_64, ppc64le, aarch64
+
+## CUDA APIs involved
+
+## Dependencies needed to build/run
+[NVRTC](../../README.md#nvrtc)
+
+## Prerequisites
+
+Download and install the [CUDA Toolkit 11.5](https://developer.nvidia.com/cuda-downloads) for your corresponding platform.
+Make sure the dependencies mentioned in [Dependencies]() section above are installed.
+
+## Build and Run
+
+### Windows
+The Windows samples are built using the Visual Studio IDE. Solution files (.sln) are provided for each supported version of Visual Studio, using the format:
+```
+*_vs<version>.sln - for Visual Studio <version>
+```
+Each individual sample has its own set of solution files in its directory:
+
+To build/examine all the samples at once, the complete solution files should be used. To build/examine a single sample, the individual sample solution files should be used.
+> **Note:** Some samples require that the Microsoft DirectX SDK (June 2010 or newer) be installed and that the VC++ directory paths are properly set up (**Tools > Options...**). Check DirectX Dependencies section for details."
+
+### Linux
+The Linux samples are built using makefiles. To use the makefiles, change the current directory to the sample directory you wish to build, and run make:
+```
+$ cd <sample_dir>
+$ make
+```
+The samples makefiles can take advantage of certain options:
+*  **TARGET_ARCH=<arch>** - cross-compile targeting a specific architecture. Allowed architectures are x86_64, ppc64le, aarch64.
+    By default, TARGET_ARCH is set to HOST_ARCH. On a x86_64 machine, not setting TARGET_ARCH is the equivalent of setting TARGET_ARCH=x86_64.<br/>
+`$ make TARGET_ARCH=x86_64` <br/> `$ make TARGET_ARCH=ppc64le` <br/> `$ make TARGET_ARCH=aarch64` <br/>
+    See [here](http://docs.nvidia.com/cuda/cuda-samples/index.html#cross-samples) for more details.
+*   **dbg=1** - build with debug symbols
+    ```
+    $ make dbg=1
+    ```
+*   **SMS="A B ..."** - override the SM architectures for which the sample will be built, where `"A B ..."` is a space-delimited list of SM architectures. For example, to generate SASS for SM 50 and SM 60, use `SMS="50 60"`.
+    ```
+    $ make SMS="50 60"
+    ```
+
+*  **HOST_COMPILER=<host_compiler>** - override the default g++ host compiler. See the [Linux Installation Guide](http://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html#system-requirements) for a list of supported host compilers.
+```
+    $ make HOST_COMPILER=g++
+```
+
+## References (for more details)
+

Samples/EGLStream_CUDA_CrossGPU/.vscode/c_cpp_properties.json

@@ -0,0 +1,18 @@
+{
+    "configurations": [
+        {
+            "name": "Linux",
+            "includePath": [
+                "${workspaceFolder}/**",
+                "${workspaceFolder}/../../Common"
+            ],
+            "defines": [],
+            "compilerPath": "/usr/local/cuda/bin/nvcc",
+            "cStandard": "gnu17",
+            "cppStandard": "gnu++14",
+            "intelliSenseMode": "linux-gcc-x64",
+            "configurationProvider": "ms-vscode.makefile-tools"
+        }
+    ],
+    "version": 4
+}

Samples/EGLStream_CUDA_CrossGPU/.vscode/extensions.json

@@ -0,0 +1,7 @@
+{
+    "recommendations": [
+        "nvidia.nsight-vscode-edition",
+        "ms-vscode.cpptools",
+        "ms-vscode.makefile-tools"
+    ]
+}

Samples/EGLStream_CUDA_CrossGPU/.vscode/launch.json

@@ -0,0 +1,10 @@
+{
+    "configurations": [
+        {
+            "name": "CUDA C++: Launch",
+            "type": "cuda-gdb",
+            "request": "launch",
+            "program": "${workspaceFolder}/EGLStream_CUDA_CrossGPU"
+        }
+    ]
+}

Samples/EGLStream_CUDA_CrossGPU/.vscode/tasks.json

@@ -0,0 +1,15 @@
+{
+    "version": "2.0.0",
+    "tasks": [
+        {
+            "label": "sample",
+            "type": "shell",
+            "command": "make dbg=1",
+            "problemMatcher": ["$nvcc"],
+            "group": {
+                "kind": "build",
+                "isDefault": true
+            }
+        }
+    ]
+}

Samples/EGLStream_CUDA_CrossGPU/Makefile

@@ -0,0 +1,453 @@
+################################################################################
+# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#  * Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#  * Neither the name of NVIDIA CORPORATION nor the names of its
+#    contributors may be used to endorse or promote products derived
+#    from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+################################################################################
+#
+# Makefile project only supported on Mac OS X and Linux Platforms)
+#
+################################################################################
+
+# Location of the CUDA Toolkit
+CUDA_PATH ?= /usr/local/cuda
+
+##############################
+# start deprecated interface #
+##############################
+ifeq ($(x86_64),1)
+    $(info WARNING - x86_64 variable has been deprecated)
+    $(info WARNING - please use TARGET_ARCH=x86_64 instead)
+    TARGET_ARCH ?= x86_64
+endif
+ifeq ($(ARMv7),1)
+    $(info WARNING - ARMv7 variable has been deprecated)
+    $(info WARNING - please use TARGET_ARCH=armv7l instead)
+    TARGET_ARCH ?= armv7l
+endif
+ifeq ($(aarch64),1)
+    $(info WARNING - aarch64 variable has been deprecated)
+    $(info WARNING - please use TARGET_ARCH=aarch64 instead)
+    TARGET_ARCH ?= aarch64
+endif
+ifeq ($(ppc64le),1)
+    $(info WARNING - ppc64le variable has been deprecated)
+    $(info WARNING - please use TARGET_ARCH=ppc64le instead)
+    TARGET_ARCH ?= ppc64le
+endif
+ifneq ($(GCC),)
+    $(info WARNING - GCC variable has been deprecated)
+    $(info WARNING - please use HOST_COMPILER=$(GCC) instead)
+    HOST_COMPILER ?= $(GCC)
+endif
+ifneq ($(abi),)
+    $(error ERROR - abi variable has been removed)
+endif
+############################
+# end deprecated interface #
+############################
+
+# architecture
+HOST_ARCH   := $(shell uname -m)
+TARGET_ARCH ?= $(HOST_ARCH)
+ifneq (,$(filter $(TARGET_ARCH),x86_64 aarch64 sbsa ppc64le armv7l))
+    ifneq ($(TARGET_ARCH),$(HOST_ARCH))
+        ifneq (,$(filter $(TARGET_ARCH),x86_64 aarch64 sbsa ppc64le))
+            TARGET_SIZE := 64
+        else ifneq (,$(filter $(TARGET_ARCH),armv7l))
+            TARGET_SIZE := 32
+        endif
+    else
+        TARGET_SIZE := $(shell getconf LONG_BIT)
+    endif
+else
+    $(error ERROR - unsupported value $(TARGET_ARCH) for TARGET_ARCH!)
+endif
+
+# sbsa and aarch64 systems look similar. Need to differentiate them at host level for now.
+ifeq ($(HOST_ARCH),aarch64)
+    ifeq ($(CUDA_PATH)/targets/sbsa-linux,$(shell ls -1d $(CUDA_PATH)/targets/sbsa-linux 2>/dev/null))
+        HOST_ARCH := sbsa
+        TARGET_ARCH := sbsa
+    endif
+endif
+
+ifneq ($(TARGET_ARCH),$(HOST_ARCH))
+    ifeq (,$(filter $(HOST_ARCH)-$(TARGET_ARCH),aarch64-armv7l x86_64-armv7l x86_64-aarch64 x86_64-sbsa x86_64-ppc64le))
+        $(error ERROR - cross compiling from $(HOST_ARCH) to $(TARGET_ARCH) is not supported!)
+    endif
+endif
+
+# When on native aarch64 system with userspace of 32-bit, change TARGET_ARCH to armv7l
+ifeq ($(HOST_ARCH)-$(TARGET_ARCH)-$(TARGET_SIZE),aarch64-aarch64-32)
+    TARGET_ARCH = armv7l
+endif
+
+# operating system
+HOST_OS   := $(shell uname -s 2>/dev/null | tr "[:upper:]" "[:lower:]")
+TARGET_OS ?= $(HOST_OS)
+ifeq (,$(filter $(TARGET_OS),linux darwin qnx android))
+    $(error ERROR - unsupported value $(TARGET_OS) for TARGET_OS!)
+endif
+
+# host compiler
+ifeq ($(TARGET_OS),darwin)
+    ifeq ($(shell expr `xcodebuild -version | grep -i xcode | awk '{print $$2}' | cut -d'.' -f1` \>= 5),1)
+        HOST_COMPILER ?= clang++
+    endif
+else ifneq ($(TARGET_ARCH),$(HOST_ARCH))
+    ifeq ($(HOST_ARCH)-$(TARGET_ARCH),x86_64-armv7l)
+        ifeq ($(TARGET_OS),linux)
+            HOST_COMPILER ?= arm-linux-gnueabihf-g++
+        else ifeq ($(TARGET_OS),qnx)
+            ifeq ($(QNX_HOST),)
+                $(error ERROR - QNX_HOST must be passed to the QNX host toolchain)
+            endif
+            ifeq ($(QNX_TARGET),)
+                $(error ERROR - QNX_TARGET must be passed to the QNX target toolchain)
+            endif
+            export QNX_HOST
+            export QNX_TARGET
+            HOST_COMPILER ?= $(QNX_HOST)/usr/bin/arm-unknown-nto-qnx6.6.0eabi-g++
+        else ifeq ($(TARGET_OS),android)
+            HOST_COMPILER ?= arm-linux-androideabi-g++
+        endif
+    else ifeq ($(TARGET_ARCH),aarch64)
+        ifeq ($(TARGET_OS), linux)
+            HOST_COMPILER ?= aarch64-linux-gnu-g++
+        else ifeq ($(TARGET_OS),qnx)
+            ifeq ($(QNX_HOST),)
+                $(error ERROR - QNX_HOST must be passed to the QNX host toolchain)
+            endif
+            ifeq ($(QNX_TARGET),)
+                $(error ERROR - QNX_TARGET must be passed to the QNX target toolchain)
+            endif
+            export QNX_HOST
+            export QNX_TARGET
+            HOST_COMPILER ?= $(QNX_HOST)/usr/bin/q++
+        else ifeq ($(TARGET_OS), android)
+            HOST_COMPILER ?= aarch64-linux-android-clang++
+        endif
+    else ifeq ($(TARGET_ARCH),sbsa)
+        HOST_COMPILER ?= aarch64-linux-gnu-g++
+    else ifeq ($(TARGET_ARCH),ppc64le)
+        HOST_COMPILER ?= powerpc64le-linux-gnu-g++
+    endif
+endif
+HOST_COMPILER ?= g++
+NVCC          := $(CUDA_PATH)/bin/nvcc -ccbin $(HOST_COMPILER)
+
+# internal flags
+NVCCFLAGS   := -m${TARGET_SIZE}
+CCFLAGS     :=
+LDFLAGS     :=
+
+# build flags
+ifeq ($(TARGET_OS),darwin)
+    LDFLAGS += -rpath $(CUDA_PATH)/lib
+    CCFLAGS += -arch $(HOST_ARCH)
+else ifeq ($(HOST_ARCH)-$(TARGET_ARCH)-$(TARGET_OS),x86_64-armv7l-linux)
+    LDFLAGS += --dynamic-linker=/lib/ld-linux-armhf.so.3
+    CCFLAGS += -mfloat-abi=hard
+else ifeq ($(TARGET_OS),android)
+    LDFLAGS += -pie
+    CCFLAGS += -fpie -fpic -fexceptions
+endif
+
+ifneq ($(TARGET_ARCH),$(HOST_ARCH))
+    ifeq ($(TARGET_ARCH)-$(TARGET_OS),armv7l-linux)
+        ifneq ($(TARGET_FS),)
+            GCCVERSIONLTEQ46 := $(shell expr `$(HOST_COMPILER) -dumpversion` \<= 4.6)
+            ifeq ($(GCCVERSIONLTEQ46),1)
+                CCFLAGS += --sysroot=$(TARGET_FS)
+            endif
+            LDFLAGS += --sysroot=$(TARGET_FS)
+            LDFLAGS += -rpath-link=$(TARGET_FS)/lib
+            LDFLAGS += -rpath-link=$(TARGET_FS)/usr/lib
+            LDFLAGS += -rpath-link=$(TARGET_FS)/usr/lib/arm-linux-gnueabihf
+        endif
+    endif
+    ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-linux)
+        ifneq ($(TARGET_FS),)
+            GCCVERSIONLTEQ46 := $(shell expr `$(HOST_COMPILER) -dumpversion` \<= 4.6)
+            ifeq ($(GCCVERSIONLTEQ46),1)
+                CCFLAGS += --sysroot=$(TARGET_FS)
+            endif
+            LDFLAGS += --sysroot=$(TARGET_FS)
+            LDFLAGS += -rpath-link=$(TARGET_FS)/lib -L$(TARGET_FS)/lib
+            LDFLAGS += -rpath-link=$(TARGET_FS)/lib/aarch64-linux-gnu -L$(TARGET_FS)/lib/aarch64-linux-gnu
+            LDFLAGS += -rpath-link=$(TARGET_FS)/usr/lib -L$(TARGET_FS)/usr/lib
+            LDFLAGS += -rpath-link=$(TARGET_FS)/usr/lib/aarch64-linux-gnu -L$(TARGET_FS)/usr/lib/aarch64-linux-gnu
+            LDFLAGS += --unresolved-symbols=ignore-in-shared-libs
+            CCFLAGS += -isystem=$(TARGET_FS)/usr/include -I$(TARGET_FS)/usr/include -I$(TARGET_FS)/usr/include/libdrm
+            CCFLAGS += -isystem=$(TARGET_FS)/usr/include/aarch64-linux-gnu -I$(TARGET_FS)/usr/include/aarch64-linux-gnu
+        endif
+    endif
+    ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-qnx)
+        NVCCFLAGS += --qpp-config 5.4.0,gcc_ntoaarch64le
+        CCFLAGS += -DWIN_INTERFACE_CUSTOM -I/usr/include/aarch64-qnx-gnu
+        LDFLAGS += -lsocket
+        LDFLAGS += -L/usr/lib/aarch64-qnx-gnu
+        CCFLAGS += "-Wl\,-rpath-link\,/usr/lib/aarch64-qnx-gnu"
+        ifdef TARGET_OVERRIDE
+            LDFLAGS += -lslog2
+        endif
+
+        ifneq ($(TARGET_FS),)
+            LDFLAGS += -L$(TARGET_FS)/usr/lib
+            CCFLAGS += "-Wl\,-rpath-link\,$(TARGET_FS)/usr/lib"
+            LDFLAGS += -L$(TARGET_FS)/usr/libnvidia
+            CCFLAGS += "-Wl\,-rpath-link\,$(TARGET_FS)/usr/libnvidia"
+            CCFLAGS += -I$(TARGET_FS)/../include
+        endif
+    endif
+endif
+
+ifdef TARGET_OVERRIDE # cuda toolkit targets override
+    NVCCFLAGS += -target-dir $(TARGET_OVERRIDE)
+endif
+
+# Install directory of different arch
+CUDA_INSTALL_TARGET_DIR :=
+ifeq ($(TARGET_ARCH)-$(TARGET_OS),armv7l-linux)
+    CUDA_INSTALL_TARGET_DIR = targets/armv7-linux-gnueabihf/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-linux)
+    CUDA_INSTALL_TARGET_DIR = targets/aarch64-linux/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),sbsa-linux)
+    CUDA_INSTALL_TARGET_DIR = targets/sbsa-linux/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),armv7l-android)
+    CUDA_INSTALL_TARGET_DIR = targets/armv7-linux-androideabi/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-android)
+    CUDA_INSTALL_TARGET_DIR = targets/aarch64-linux-androideabi/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),armv7l-qnx)
+    CUDA_INSTALL_TARGET_DIR = targets/ARMv7-linux-QNX/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-qnx)
+    CUDA_INSTALL_TARGET_DIR = targets/aarch64-qnx/
+else ifeq ($(TARGET_ARCH),ppc64le)
+    CUDA_INSTALL_TARGET_DIR = targets/ppc64le-linux/
+endif
+
+# Debug build flags
+ifeq ($(dbg),1)
+      NVCCFLAGS += -g -G
+      BUILD_TYPE := debug
+else
+      BUILD_TYPE := release
+endif
+
+ALL_CCFLAGS :=
+ALL_CCFLAGS += $(NVCCFLAGS)
+ALL_CCFLAGS += $(EXTRA_NVCCFLAGS)
+ALL_CCFLAGS += $(addprefix -Xcompiler ,$(CCFLAGS))
+ALL_CCFLAGS += $(addprefix -Xcompiler ,$(EXTRA_CCFLAGS))
+
+UBUNTU = $(shell lsb_release -i -s 2>/dev/null | grep -i ubuntu)
+
+SAMPLE_ENABLED := 1
+
+# This sample is not supported on Mac OSX
+ifeq ($(TARGET_OS),darwin)
+  $(info >>> WARNING - EGLStream_CUDA_CrossGPU is not supported on Mac OSX - waiving sample <<<)
+  SAMPLE_ENABLED := 0
+endif
+
+# This sample is not supported on android
+ifeq ($(TARGET_OS),android)
+  $(info >>> WARNING - EGLStream_CUDA_CrossGPU is not supported on android - waiving sample <<<)
+  SAMPLE_ENABLED := 0
+endif
+
+ALL_LDFLAGS :=
+ALL_LDFLAGS += $(ALL_CCFLAGS)
+ALL_LDFLAGS += $(addprefix -Xlinker ,$(LDFLAGS))
+ALL_LDFLAGS += $(addprefix -Xlinker ,$(EXTRA_LDFLAGS))
+
+# Common includes and paths for CUDA
+INCLUDES  := -I../../Common
+LIBRARIES :=
+
+################################################################################
+
+# Makefile include to help find EGL Libraries
+include ./findegl.mk
+
+# EGL specific libraries
+ifneq ($(TARGET_OS),darwin)
+ LIBRARIES += -lEGL
+endif
+
+#Detect if installed version of GCC supports required C++11
+ifeq ($(TARGET_OS),linux)
+    empty :=
+    space := $(empty) $(empty)
+    GCCVERSIONSTRING := $(shell expr `$(HOST_COMPILER) -dumpversion`)
+#Create version number without "."
+    GCCVERSION := $(shell expr `echo $(GCCVERSIONSTRING)` | cut -f1 -d.)
+    GCCVERSION += $(shell expr `echo $(GCCVERSIONSTRING)` | cut -f2 -d.)
+    GCCVERSION += $(shell expr `echo $(GCCVERSIONSTRING)` | cut -f3 -d.)
+# Make sure the version number has at least 3 decimals
+    GCCVERSION += 00
+# Remove spaces from the version number
+    GCCVERSION := $(subst $(space),$(empty),$(GCCVERSION))
+#$(warning $(GCCVERSION))
+
+    IS_MIN_VERSION := $(shell expr `echo $(GCCVERSION)` \>= 51000)
+
+    ifeq ($(IS_MIN_VERSION), 1)
+        $(info >>> GCC Version is greater or equal to 5.1.0 <<<)
+    else
+        $(info >>> Waiving build. Minimum GCC version required is 5.1.0<<<)
+        SAMPLE_ENABLED := 0
+    endif
+endif
+
+# Gencode arguments
+ifeq ($(TARGET_ARCH),$(filter $(TARGET_ARCH),armv7l aarch64))
+SMS ?= 35 37 50 52 60 61 70 72 75 80 86
+else
+SMS ?= 35 37 50 52 60 61 70 75 80 86
+endif
+
+ifeq ($(SMS),)
+$(info >>> WARNING - no SM architectures have been specified - waiving sample <<<)
+SAMPLE_ENABLED := 0
+endif
+
+ifeq ($(GENCODE_FLAGS),)
+# Generate SASS code for each SM architecture listed in $(SMS)
+$(foreach sm,$(SMS),$(eval GENCODE_FLAGS += -gencode arch=compute_$(sm),code=sm_$(sm)))
+
+# Generate PTX code from the highest SM architecture in $(SMS) to guarantee forward-compatibility
+HIGHEST_SM := $(lastword $(sort $(SMS)))
+ifneq ($(HIGHEST_SM),)
+GENCODE_FLAGS += -gencode arch=compute_$(HIGHEST_SM),code=compute_$(HIGHEST_SM)
+endif
+endif
+
+ifeq ($(TARGET_OS),darwin)
+  ALL_LDFLAGS += -Xcompiler -F/Library/Frameworks -Xlinker -framework -Xlinker CUDA
+else
+  ifeq ($(TARGET_ARCH),x86_64)
+    CUDA_SEARCH_PATH ?= $(CUDA_PATH)/lib64/stubs
+    CUDA_SEARCH_PATH += $(CUDA_PATH)/targets/x86_64-linux/lib/stubs
+  endif
+
+  ifeq ($(TARGET_ARCH)-$(TARGET_OS),armv7l-linux)
+    CUDA_SEARCH_PATH ?= $(CUDA_PATH)/targets/armv7-linux-gnueabihf/lib/stubs
+  endif
+
+  ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-linux)
+    CUDA_SEARCH_PATH ?= $(CUDA_PATH)/targets/aarch64-linux/lib/stubs
+  endif
+
+  ifeq ($(TARGET_ARCH)-$(TARGET_OS),sbsa-linux)
+    CUDA_SEARCH_PATH ?= $(CUDA_PATH)/targets/sbsa-linux/lib/stubs
+  endif
+
+  ifeq ($(TARGET_ARCH)-$(TARGET_OS),armv7l-android)
+    CUDA_SEARCH_PATH ?= $(CUDA_PATH)/targets/armv7-linux-androideabi/lib/stubs
+  endif
+
+  ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-android)
+    CUDA_SEARCH_PATH ?= $(CUDA_PATH)/targets/aarch64-linux-androideabi/lib/stubs
+  endif
+
+  ifeq ($(TARGET_ARCH)-$(TARGET_OS),armv7l-qnx)
+    CUDA_SEARCH_PATH ?= $(CUDA_PATH)/targets/ARMv7-linux-QNX/lib/stubs
+  endif
+
+  ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-qnx)
+    CUDA_SEARCH_PATH ?= $(CUDA_PATH)/targets/aarch64-qnx/lib/stubs
+    ifdef TARGET_OVERRIDE
+        CUDA_SEARCH_PATH := $(CUDA_PATH)/targets/$(TARGET_OVERRIDE)/lib/stubs
+    endif
+  endif
+
+  ifeq ($(TARGET_ARCH),ppc64le)
+    CUDA_SEARCH_PATH ?= $(CUDA_PATH)/targets/ppc64le-linux/lib/stubs
+  endif
+
+  ifeq ($(HOST_ARCH),ppc64le)
+    CUDA_SEARCH_PATH += $(CUDA_PATH)/lib64/stubs
+  endif
+
+  CUDALIB ?= $(shell find -L $(CUDA_SEARCH_PATH) -maxdepth 1 -name libcuda.so 2> /dev/null)
+  ifeq ("$(CUDALIB)","")
+    $(info >>> WARNING - libcuda.so not found, CUDA Driver is not installed.  Please re-install the driver. <<<)
+    SAMPLE_ENABLED := 0
+  else
+    CUDALIB := $(shell echo $(CUDALIB) | sed "s/ .*//" | sed "s/\/libcuda.so//" )
+    LIBRARIES += -L$(CUDALIB) -lcuda
+  endif
+endif
+
+ALL_CCFLAGS += --threads 0 --std=c++11
+
+ifeq ($(SAMPLE_ENABLED),0)
+EXEC ?= @echo "[@]"
+endif
+
+################################################################################
+
+# Target rules
+all: build
+
+build: EGLStream_CUDA_CrossGPU
+
+check.deps:
+ifeq ($(SAMPLE_ENABLED),0)
+	@echo "Sample will be waived due to the above missing dependencies"
+else
+	@echo "Sample is ready - all dependencies have been met"
+endif
+
+cuda_consumer.o:cuda_consumer.cpp
+	$(EXEC) $(NVCC) $(INCLUDES) $(ALL_CCFLAGS) $(GENCODE_FLAGS) -o $@ -c $<
+
+cuda_producer.o:cuda_producer.cpp
+	$(EXEC) $(NVCC) $(INCLUDES) $(ALL_CCFLAGS) $(GENCODE_FLAGS) -o $@ -c $<
+
+eglstrm_common.o:eglstrm_common.cpp
+	$(EXEC) $(NVCC) $(INCLUDES) $(ALL_CCFLAGS) $(GENCODE_FLAGS) -o $@ -c $<
+
+kernel.o:kernel.cu
+	$(EXEC) $(NVCC) $(INCLUDES) $(ALL_CCFLAGS) $(GENCODE_FLAGS) -o $@ -c $<
+
+main.o:main.cpp
+	$(EXEC) $(NVCC) $(INCLUDES) $(ALL_CCFLAGS) $(GENCODE_FLAGS) -o $@ -c $<
+
+EGLStream_CUDA_CrossGPU: cuda_consumer.o cuda_producer.o eglstrm_common.o kernel.o main.o
+	$(EXEC) $(NVCC) $(ALL_LDFLAGS) $(GENCODE_FLAGS) -o $@ $+ $(LIBRARIES)
+	$(EXEC) mkdir -p ../../bin/$(TARGET_ARCH)/$(TARGET_OS)/$(BUILD_TYPE)
+	$(EXEC) cp $@ ../../bin/$(TARGET_ARCH)/$(TARGET_OS)/$(BUILD_TYPE)
+
+run: build
+	$(EXEC) ./EGLStream_CUDA_CrossGPU
+
+clean:
+	rm -f EGLStream_CUDA_CrossGPU cuda_consumer.o cuda_producer.o eglstrm_common.o kernel.o main.o
+	rm -rf ../../bin/$(TARGET_ARCH)/$(TARGET_OS)/$(BUILD_TYPE)/EGLStream_CUDA_CrossGPU
+
+clobber: clean

Samples/EGLStream_CUDA_CrossGPU/NsightEclipse.xml

@@ -0,0 +1,78 @@
+<?xml version="1.0" encoding="UTF-8"?> 
+<!DOCTYPE entry SYSTEM "SamplesInfo.dtd">
+<entry>
+  <name>EGLStream_CUDA_CrossGPU</name>
+  <cuda_api_list>
+    <driver>cuDeviceGet</driver>
+    <driver>cuDeviceGetAttribute</driver>
+    <driver>cuDeviceComputeCapability</driver>
+    <driver>cuDeviceGetCount</driver>
+    <driver>cuDeviceGetName</driver>
+    <driver>cuGraphicsResourceGetMappedEglFrame</driver>
+    <driver>cuEGLStreamConsumerAcquireFrame</driver>
+    <driver>cuEGLStreamConsumerReleaseFrame</driver>
+    <driver>cuEGLStreamProducerReturnFrame</driver>
+    <driver>cuEGLStreamProducerPresentFrame</driver>
+    <driver>cuCtxCreate</driver>
+    <driver>cuMemAlloc</driver>
+    <driver>cuMemFree</driver>
+    <driver>cuMemcpy3D</driver>
+    <driver>cuStreamCreate</driver>
+    <driver>cuCtxPushCurrent</driver>
+    <driver>cuCtxPopCurrent</driver>
+    <driver>cuCtxDestroy</driver>
+  </cuda_api_list>
+  <description><![CDATA[Demonstrates CUDA and EGL Streams interop, where consumer's EGL Stream is on one GPU and producer's on other and both consumer-producer are different processes.]]></description>
+  <devicecompilation>whole</devicecompilation>
+  <includepaths>
+    <path>./</path>
+    <path>../</path>
+    <path>../../Common</path>
+  </includepaths>
+  <keyconcepts>
+    <concept level="basic">EGLStreams Interop</concept>
+  </keyconcepts>
+  <keywords>
+    <keyword>EGL Streams</keyword>
+  </keywords>
+  <libraries>
+    <library os="linux">cuda</library>
+    <library framework="true" os="macosx">CUDA</library>
+  </libraries>
+  <librarypaths>
+  </librarypaths>
+  <nsight_eclipse>true</nsight_eclipse>
+  <primary_file>main.cpp</primary_file>
+  <required_dependencies>
+    <dependency>EGL</dependency>
+  </required_dependencies>
+  <scopes>
+    <scope>1:CUDA Advanced Topics</scope>
+    <scope>2:Graphics Interop</scope>
+  </scopes>
+  <sm-arch>sm35</sm-arch>
+  <sm-arch>sm37</sm-arch>
+  <sm-arch>sm50</sm-arch>
+  <sm-arch>sm52</sm-arch>
+  <sm-arch>sm60</sm-arch>
+  <sm-arch>sm61</sm-arch>
+  <sm-arch>sm70</sm-arch>
+  <sm-arch>sm72</sm-arch>
+  <sm-arch>sm75</sm-arch>
+  <sm-arch>sm80</sm-arch>
+  <sm-arch>sm86</sm-arch>
+  <supported_envs>
+    <env>
+      <arch>x86_64</arch>
+      <platform>linux</platform>
+    </env>
+    <env>
+      <arch>arm</arch>
+    </env>
+  </supported_envs>
+  <supported_sm_architectures>
+    <include>all</include>
+  </supported_sm_architectures>
+  <title>EGLStream_CUDA_CrossGPU</title>
+  <type>exe</type>
+</entry>

Samples/EGLStream_CUDA_CrossGPU/README.md

@@ -0,0 +1,64 @@
+# EGLStream_CUDA_CrossGPU - EGLStream_CUDA_CrossGPU
+
+## Description
+
+Demonstrates CUDA and EGL Streams interop, where consumer's EGL Stream is on one GPU and producer's on other and both consumer-producer are different processes.
+
+## Key Concepts
+
+EGLStreams Interop
+
+## Supported SM Architectures
+
+[SM 3.5 ](https://developer.nvidia.com/cuda-gpus)  [SM 3.7 ](https://developer.nvidia.com/cuda-gpus)  [SM 5.0 ](https://developer.nvidia.com/cuda-gpus)  [SM 5.2 ](https://developer.nvidia.com/cuda-gpus)  [SM 6.0 ](https://developer.nvidia.com/cuda-gpus)  [SM 6.1 ](https://developer.nvidia.com/cuda-gpus)  [SM 7.0 ](https://developer.nvidia.com/cuda-gpus)  [SM 7.2 ](https://developer.nvidia.com/cuda-gpus)  [SM 7.5 ](https://developer.nvidia.com/cuda-gpus)  [SM 8.0 ](https://developer.nvidia.com/cuda-gpus)  [SM 8.6 ](https://developer.nvidia.com/cuda-gpus)
+
+## Supported OSes
+
+Linux
+
+## Supported CPU Architecture
+
+x86_64, armv7l
+
+## CUDA APIs involved
+
+### [CUDA Driver API](http://docs.nvidia.com/cuda/cuda-driver-api/index.html)
+cuDeviceGet, cuDeviceGetAttribute, cuDeviceComputeCapability, cuDeviceGetCount, cuDeviceGetName, cuGraphicsResourceGetMappedEglFrame, cuEGLStreamConsumerAcquireFrame, cuEGLStreamConsumerReleaseFrame, cuEGLStreamProducerReturnFrame, cuEGLStreamProducerPresentFrame, cuCtxCreate, cuMemAlloc, cuMemFree, cuMemcpy3D, cuStreamCreate, cuCtxPushCurrent, cuCtxPopCurrent, cuCtxDestroy
+
+## Dependencies needed to build/run
+[EGL](../../README.md#egl)
+
+## Prerequisites
+
+Download and install the [CUDA Toolkit 11.5](https://developer.nvidia.com/cuda-downloads) for your corresponding platform.
+Make sure the dependencies mentioned in [Dependencies]() section above are installed.
+
+## Build and Run
+
+### Linux
+The Linux samples are built using makefiles. To use the makefiles, change the current directory to the sample directory you wish to build, and run make:
+```
+$ cd <sample_dir>
+$ make
+```
+The samples makefiles can take advantage of certain options:
+*  **TARGET_ARCH=<arch>** - cross-compile targeting a specific architecture. Allowed architectures are x86_64, armv7l.
+    By default, TARGET_ARCH is set to HOST_ARCH. On a x86_64 machine, not setting TARGET_ARCH is the equivalent of setting TARGET_ARCH=x86_64.<br/>
+`$ make TARGET_ARCH=x86_64` <br/> `$ make TARGET_ARCH=armv7l` <br/>
+    See [here](http://docs.nvidia.com/cuda/cuda-samples/index.html#cross-samples) for more details.
+*   **dbg=1** - build with debug symbols
+    ```
+    $ make dbg=1
+    ```
+*   **SMS="A B ..."** - override the SM architectures for which the sample will be built, where `"A B ..."` is a space-delimited list of SM architectures. For example, to generate SASS for SM 50 and SM 60, use `SMS="50 60"`.
+    ```
+    $ make SMS="50 60"
+    ```
+
+*  **HOST_COMPILER=<host_compiler>** - override the default g++ host compiler. See the [Linux Installation Guide](http://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html#system-requirements) for a list of supported host compilers.
+```
+    $ make HOST_COMPILER=g++
+```
+
+## References (for more details)
+

Samples/EGLStream_CUDA_CrossGPU/cuda_consumer.cpp

@@ -0,0 +1,258 @@
+/* Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+//
+// DESCRIPTION:   Simple CUDA consumer rendering sample app
+//
+
+#include <cuda_runtime.h>
+#include "cuda_consumer.h"
+#include "eglstrm_common.h"
+#include <math.h>
+#include <unistd.h>
+
+#if defined(EXTENSION_LIST)
+EXTENSION_LIST(EXTLST_EXTERN)
+#endif
+CUgraphicsResource cudaResource;
+
+static int count_acq = 0;
+static double acquire_time[25000] = {0}, total_time_acq = 0;
+
+static int count_rel = 0;
+static double rel_time[25000] = {0}, total_time_rel = 0;
+
+void acquireApiStat(void);
+void acquireApiStat(void) {
+  int i = 0;
+  double min = 10000000, max = 0;
+  double average_launch_time = 0, standard_deviation = 0;
+  if (count_acq == 0) return;
+  // lets compute the standard deviation
+  min = max = acquire_time[1];
+  average_launch_time = (total_time_acq - acquire_time[0]) / count_acq;
+  for (i = 1; i < count_acq; i++) {
+    standard_deviation += (acquire_time[i] - average_launch_time) *
+                          (acquire_time[i] - average_launch_time);
+    if (acquire_time[i] < min) min = acquire_time[i];
+    if (acquire_time[i] > max) max = acquire_time[i];
+  }
+  standard_deviation = sqrt(standard_deviation / count_acq);
+  printf("acquire Avg: %lf\n", average_launch_time);
+  printf("acquire  SD: %lf\n", standard_deviation);
+  printf("acquire min: %lf\n", min);
+  printf("acquire max: %lf\n", max);
+
+  min = max = rel_time[1];
+  average_launch_time = (total_time_rel - rel_time[0]) / count_rel;
+  for (i = 1; i < count_rel; i++) {
+    standard_deviation += (rel_time[i] - average_launch_time) *
+                          (rel_time[i] - average_launch_time);
+    if (rel_time[i] < min) min = rel_time[i];
+    if (rel_time[i] > max) max = rel_time[i];
+  }
+  standard_deviation = sqrt(standard_deviation / count_rel);
+  printf("release Avg: %lf\n", average_launch_time);
+  printf("release  SD: %lf\n", standard_deviation);
+  printf("release min: %lf\n", min);
+  printf("release max: %lf\n", max);
+}
+CUresult cudaConsumerAcquireFrame(test_cuda_consumer_s *cudaConsumer,
+                                  int frameNumber) {
+  CUresult cuStatus = CUDA_SUCCESS;
+  CUeglFrame cudaEgl;
+  struct timespec start, end;
+  EGLint streamState = 0;
+  double curTime;
+
+  if (!cudaConsumer) {
+    printf("%s: Bad parameter\n", __func__);
+    goto done;
+  }
+
+  while (1) {
+    if (!eglQueryStreamKHR(cudaConsumer->eglDisplay, cudaConsumer->eglStream,
+                           EGL_STREAM_STATE_KHR, &streamState)) {
+      printf("Cuda Consumer: eglQueryStreamKHR EGL_STREAM_STATE_KHR failed\n");
+      cuStatus = CUDA_ERROR_UNKNOWN;
+      goto done;
+    }
+    if (streamState == EGL_STREAM_STATE_DISCONNECTED_KHR) {
+      printf("Cuda Consumer: EGL_STREAM_STATE_DISCONNECTED_KHR received\n");
+      cuStatus = CUDA_ERROR_UNKNOWN;
+      goto done;
+    }
+
+    if (streamState == EGL_STREAM_STATE_NEW_FRAME_AVAILABLE_KHR) {
+      break;
+    }
+  }
+  if (cudaConsumer->profileAPI) {
+    getTime(&start);
+  }
+  cuStatus =
+      cuEGLStreamConsumerAcquireFrame(&(cudaConsumer->cudaConn), &cudaResource,
+                                      &cudaConsumer->consCudaStream, 16000);
+  if (cudaConsumer->profileAPI) {
+    getTime(&end);
+    curTime = TIME_DIFF(end, start);
+    acquire_time[count_acq++] = curTime;
+    if (count_acq == 25000) count_acq = 0;
+    total_time_acq += curTime;
+  }
+  if (cuStatus == CUDA_SUCCESS) {
+    CUdeviceptr pDevPtr = 0;
+    cudaError_t err;
+
+    cuStatus =
+        cuGraphicsResourceGetMappedEglFrame(&cudaEgl, cudaResource, 0, 0);
+    if (cuStatus != CUDA_SUCCESS) {
+      printf("Cuda get resource failed with %d\n", cuStatus);
+      goto done;
+    }
+    pDevPtr = (CUdeviceptr)cudaEgl.frame.pPitch[0];
+
+    err = cudaConsumer_filter(cudaConsumer->consCudaStream, (char *)pDevPtr,
+                              WIDTH * 4, HEIGHT, PROD_DATA + frameNumber,
+                              CONS_DATA + frameNumber, frameNumber);
+    if (err != cudaSuccess) {
+      printf("Cuda Consumer: kernel failed with: %s\n",
+             cudaGetErrorString(err));
+      goto done;
+    }
+  }
+
+done:
+  return cuStatus;
+}
+
+CUresult cudaConsumerReleaseFrame(test_cuda_consumer_s *cudaConsumer,
+                                  int frameNumber) {
+  CUresult cuStatus = CUDA_SUCCESS;
+  struct timespec start, end;
+  double curTime;
+
+  if (!cudaConsumer) {
+    printf("%s: Bad parameter\n", __func__);
+    goto done;
+  }
+  if (cudaConsumer->profileAPI) {
+    getTime(&start);
+  }
+  cuStatus = cuEGLStreamConsumerReleaseFrame(
+      &cudaConsumer->cudaConn, cudaResource, &cudaConsumer->consCudaStream);
+  if (cudaConsumer->profileAPI) {
+    getTime(&end);
+    curTime = TIME_DIFF(end, start);
+    rel_time[count_rel++] = curTime;
+    if (count_rel == 25000) count_rel = 0;
+    total_time_rel += curTime;
+  }
+  if (cuStatus != CUDA_SUCCESS) {
+    printf("cuEGLStreamConsumerReleaseFrame failed, status:%d\n", cuStatus);
+    goto done;
+  }
+
+done:
+  return cuStatus;
+}
+
+CUresult cudaDeviceCreateConsumer(test_cuda_consumer_s *cudaConsumer) {
+  CUdevice device;
+  CUresult status = CUDA_SUCCESS;
+
+  if (CUDA_SUCCESS != (status = cuInit(0))) {
+    printf("Failed to initialize CUDA\n");
+    return status;
+  }
+
+  if (CUDA_SUCCESS !=
+      (status = cuDeviceGet(&device, cudaConsumer->cudaDevId))) {
+    printf("failed to get CUDA device\n");
+    return status;
+  }
+
+  if (CUDA_SUCCESS !=
+      (status = cuCtxCreate(&cudaConsumer->context, 0, device))) {
+    printf("failed to create CUDA context\n");
+    return status;
+  }
+
+  int major = 0, minor = 0;
+  char deviceName[256];
+  cuDeviceGetAttribute(&major, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR,
+                       device);
+  cuDeviceGetAttribute(&minor, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR,
+                       device);
+  cuDeviceGetName(deviceName, 256, device);
+  printf(
+      "CUDA Consumer on GPU Device %d: \"%s\" with compute capability "
+      "%d.%d\n\n",
+      device, deviceName, major, minor);
+
+  cuCtxPopCurrent(&cudaConsumer->context);
+  if (major < 6) {
+    printf(
+        "EGLStream_CUDA_CrossGPU requires SM 6.0 or higher arch GPU.  "
+        "Exiting...\n");
+    exit(2);  // EXIT_WAIVED
+  }
+
+  return status;
+}
+
+CUresult cuda_consumer_init(test_cuda_consumer_s *cudaConsumer,
+                            TestArgs *args) {
+  CUresult status = CUDA_SUCCESS;
+  int bufferSize;
+
+  cudaConsumer->charCnt = args->charCnt;
+  bufferSize = args->charCnt;
+
+  cudaConsumer->pCudaCopyMem = (unsigned char *)malloc(bufferSize);
+  if (cudaConsumer->pCudaCopyMem == NULL) {
+    printf("Cuda Consumer: malloc failed\n");
+    goto done;
+  }
+
+  status = cuStreamCreate(&cudaConsumer->consCudaStream, 0);
+  if (status != CUDA_SUCCESS) {
+    printf("Cuda Consumer: cuStreamCreate failed, status:%d\n", status);
+    goto done;
+  }
+
+  atexit(acquireApiStat);
+done:
+  return status;
+}
+
+CUresult cuda_consumer_Deinit(test_cuda_consumer_s *cudaConsumer) {
+  if (cudaConsumer->pCudaCopyMem) {
+    free(cudaConsumer->pCudaCopyMem);
+  }
+  return cuEGLStreamConsumerDisconnect(&cudaConsumer->cudaConn);
+}

Samples/EGLStream_CUDA_CrossGPU/cuda_consumer.h

@@ -0,0 +1,66 @@
+/* Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+//
+// DESCRIPTION:   CUDA consumer header file
+//
+
+#ifndef _CUDA_CONSUMER_H_
+#define _CUDA_CONSUMER_H_
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include "cudaEGL.h"
+#include "eglstrm_common.h"
+#include <cuda_runtime.h>
+#include <cuda.h>
+
+typedef struct _test_cuda_consumer_s {
+  CUcontext context;
+  CUeglStreamConnection cudaConn;
+  int cudaDevId;
+  EGLDisplay eglDisplay;
+  EGLStreamKHR eglStream;
+  unsigned int charCnt;
+  char *cudaBuf;
+  bool profileAPI;
+  unsigned char *pCudaCopyMem;
+  CUstream consCudaStream;
+} test_cuda_consumer_s;
+
+CUresult cuda_consumer_init(test_cuda_consumer_s *cudaConsumer, TestArgs *args);
+CUresult cuda_consumer_Deinit(test_cuda_consumer_s *cudaConsumer);
+CUresult cudaConsumerAcquireFrame(test_cuda_consumer_s *data, int frameNumber);
+CUresult cudaConsumerReleaseFrame(test_cuda_consumer_s *data, int frameNumber);
+CUresult cudaDeviceCreateConsumer(test_cuda_consumer_s *cudaConsumer);
+cudaError_t cudaConsumer_filter(CUstream cStream, char *pSrc, int width,
+                                int height, char expectedVal, char newVal,
+                                int frameNumber);
+cudaError_t cudaGetValueMismatch(void);
+
+#endif

Samples/EGLStream_CUDA_CrossGPU/cuda_producer.cpp

@@ -0,0 +1,288 @@
+/* Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+//
+// DESCRIPTION:   Simple cuda EGL stream producer app
+//
+
+#include "cudaEGL.h"
+#include "cuda_producer.h"
+#include "eglstrm_common.h"
+#include <cuda_runtime.h>
+#if defined(EXTENSION_LIST)
+EXTENSION_LIST(EXTLST_EXTERN)
+#endif
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <string.h>
+#include "cuda_runtime.h"
+#include "math.h"
+
+int cudaPresentReturnData = INIT_DATA;
+int fakePresent = 0;
+CUeglFrame fakeFrame;
+CUdeviceptr cudaPtrFake;
+extern bool isCrossDevice;
+
+void cudaProducerPrepareFrame(CUeglFrame *cudaEgl, CUdeviceptr cudaPtr,
+                              int bufferSize) {
+  cudaEgl->frame.pPitch[0] = (void *)cudaPtr;
+  cudaEgl->width = WIDTH;
+  cudaEgl->depth = 0;
+  cudaEgl->height = HEIGHT;
+  cudaEgl->pitch = WIDTH * 4;
+  cudaEgl->frameType = CU_EGL_FRAME_TYPE_PITCH;
+  cudaEgl->planeCount = 1;
+  cudaEgl->numChannels = 4;
+  cudaEgl->eglColorFormat = CU_EGL_COLOR_FORMAT_ARGB;
+  cudaEgl->cuFormat = CU_AD_FORMAT_UNSIGNED_INT8;
+}
+
+static int count_present = 0, count_return = 0;
+static double present_time[25000] = {0}, total_time_present = 0;
+static double return_time[25000] = {0}, total_time_return = 0;
+
+void presentApiStat(void);
+void presentApiStat(void) {
+  int i = 0;
+  double min = 10000000, max = 0;
+  double average_launch_time = 0, standard_deviation = 0;
+  if (count_present == 0) return;
+  // lets compute the standard deviation
+  min = max = present_time[1];
+  average_launch_time = (total_time_present) / count_present;
+  for (i = 1; i < count_present; i++) {
+    standard_deviation += (present_time[i] - average_launch_time) *
+                          (present_time[i] - average_launch_time);
+    if (present_time[i] < min) min = present_time[i];
+    if (present_time[i] > max) max = present_time[i];
+  }
+  standard_deviation = sqrt(standard_deviation / count_present);
+  printf("present Avg: %lf\n", average_launch_time);
+  printf("present  SD: %lf\n", standard_deviation);
+  printf("present min: %lf\n", min);
+  printf("present max: %lf\n", max);
+
+  min = max = return_time[1];
+  average_launch_time = (total_time_return - return_time[0]) / count_return;
+  for (i = 1; i < count_return; i++) {
+    standard_deviation += (return_time[i] - average_launch_time) *
+                          (return_time[i] - average_launch_time);
+    if (return_time[i] < min) min = return_time[i];
+    if (return_time[i] > max) max = return_time[i];
+  }
+  standard_deviation = sqrt(standard_deviation / count_return);
+  printf("return  Avg: %lf\n", average_launch_time);
+  printf("return   SD: %lf\n", standard_deviation);
+  printf("return  min: %lf\n", min);
+  printf("return  max: %lf\n", max);
+}
+CUresult cudaProducerPresentFrame(test_cuda_producer_s *cudaProducer,
+                                  CUeglFrame cudaEgl, int t) {
+  static int flag = 0;
+  CUresult status = CUDA_SUCCESS;
+  struct timespec start, end;
+  double curTime;
+  CUdeviceptr pDevPtr = (CUdeviceptr)cudaEgl.frame.pPitch[0];
+  cudaProducer_filter(cudaProducer->prodCudaStream, (char *)pDevPtr, WIDTH * 4,
+                      HEIGHT, cudaPresentReturnData, PROD_DATA + t, t);
+  if (cudaProducer->profileAPI) {
+    getTime(&start);
+  }
+  status = cuEGLStreamProducerPresentFrame(&cudaProducer->cudaConn, cudaEgl,
+                                           &cudaProducer->prodCudaStream);
+  if (status != CUDA_SUCCESS) {
+    printf("Cuda Producer: Present frame failed, status:%d\n", status);
+    goto done;
+  }
+  flag++;
+  if (cudaProducer->profileAPI && flag > 10) {
+    getTime(&end);
+    curTime = TIME_DIFF(end, start);
+    present_time[count_present++] = curTime;
+    if (count_present == 25000) count_present = 0;
+    total_time_present += curTime;
+  }
+done:
+  return status;
+}
+
+int flag = 0;
+CUresult cudaProducerReturnFrame(test_cuda_producer_s *cudaProducer,
+                                 CUeglFrame cudaEgl, int t) {
+  CUresult status = CUDA_SUCCESS;
+  struct timespec start, end;
+  double curTime;
+  CUdeviceptr pDevPtr = 0;
+
+  pDevPtr = (CUdeviceptr)cudaEgl.frame.pPitch[0];
+  if (cudaProducer->profileAPI) {
+    getTime(&start);
+  }
+
+  while (1) {
+    status = cuEGLStreamProducerReturnFrame(&cudaProducer->cudaConn, &cudaEgl,
+                                            &cudaProducer->prodCudaStream);
+    if (status == CUDA_ERROR_LAUNCH_TIMEOUT) {
+      continue;
+    } else if (status != CUDA_SUCCESS) {
+      printf("Cuda Producer: Return frame failed, status:%d\n", status);
+      goto done;
+    }
+    break;
+  }
+  if (cudaProducer->profileAPI) {
+    getTime(&end);
+    curTime = TIME_DIFF(end, start);
+    return_time[count_return++] = curTime;
+    if (count_return == 25000) count_return = 0;
+    total_time_return += curTime;
+  }
+  if (flag % 2 == 0) {
+    cudaPresentReturnData++;
+  }
+  cudaProducer_filter(cudaProducer->prodCudaStream, (char *)pDevPtr, WIDTH * 4,
+                      HEIGHT, CONS_DATA + t, cudaPresentReturnData, t);
+  flag++;
+done:
+  return status;
+}
+
+CUresult cudaDeviceCreateProducer(test_cuda_producer_s *cudaProducer) {
+  CUdevice device;
+  CUresult status = CUDA_SUCCESS;
+
+  if (CUDA_SUCCESS != (status = cuInit(0))) {
+    printf("Failed to initialize CUDA\n");
+    return status;
+  }
+
+  if (CUDA_SUCCESS !=
+      (status = cuDeviceGet(&device, cudaProducer->cudaDevId))) {
+    printf("failed to get CUDA device\n");
+    return status;
+  }
+
+  if (CUDA_SUCCESS !=
+      (status = cuCtxCreate(&cudaProducer->context, 0, device))) {
+    printf("failed to create CUDA context\n");
+    return status;
+  }
+
+  int major = 0, minor = 0;
+  char deviceName[256];
+  cuDeviceGetAttribute(&major, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR,
+                       device);
+  cuDeviceGetAttribute(&minor, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR,
+                       device);
+  cuDeviceGetName(deviceName, 256, device);
+  printf(
+      "CUDA Producer on GPU Device %d: \"%s\" with compute capability "
+      "%d.%d\n\n",
+      device, deviceName, major, minor);
+
+  cuCtxPopCurrent(&cudaProducer->context);
+
+  if (major < 6) {
+    printf(
+        "EGLStream_CUDA_CrossGPU requires SM 6.0 or higher arch GPU.  "
+        "Exiting...\n");
+    exit(2);  // EXIT_WAIVED
+  }
+
+  return status;
+}
+
+CUresult cudaProducerInit(test_cuda_producer_s *cudaProducer, TestArgs *args) {
+  CUresult status = CUDA_SUCCESS;
+  int bufferSize;
+
+  cudaProducer->charCnt = args->charCnt;
+  bufferSize = cudaProducer->charCnt;
+
+  cudaProducer->tempBuff = (char *)malloc(bufferSize);
+  if (!cudaProducer->tempBuff) {
+    printf("Cuda Producer: Failed to allocate image buffer\n");
+    status = CUDA_ERROR_UNKNOWN;
+    goto done;
+  }
+  memset((void *)cudaProducer->tempBuff, INIT_DATA, cudaProducer->charCnt);
+
+  // Fill this init data
+  status = cuMemAlloc(&cudaProducer->cudaPtr, bufferSize);
+  if (status != CUDA_SUCCESS) {
+    printf("Cuda Producer: cuda Malloc failed, status:%d\n", status);
+    goto done;
+  }
+  status = cuMemcpyHtoD(cudaProducer->cudaPtr, (void *)(cudaProducer->tempBuff),
+                        bufferSize);
+  if (status != CUDA_SUCCESS) {
+    printf("Cuda Producer: cuMemCpy failed, status:%d\n", status);
+    goto done;
+  }
+
+  // Fill this init data
+  status = cuMemAlloc(&cudaProducer->cudaPtr1, bufferSize);
+  if (status != CUDA_SUCCESS) {
+    printf("Cuda Producer: cuda Malloc failed, status:%d\n", status);
+    goto done;
+  }
+  status = cuMemcpyHtoD(cudaProducer->cudaPtr1,
+                        (void *)(cudaProducer->tempBuff), bufferSize);
+  if (status != CUDA_SUCCESS) {
+    printf("Cuda Producer: cuMemCpy failed, status:%d\n", status);
+    goto done;
+  }
+
+  status = cuStreamCreate(&cudaProducer->prodCudaStream, 0);
+  if (status != CUDA_SUCCESS) {
+    printf("Cuda Producer: cuStreamCreate failed, status:%d\n", status);
+    goto done;
+  }
+
+  // Fill this init data
+  status = cuMemAlloc(&cudaPtrFake, 100);
+  if (status != CUDA_SUCCESS) {
+    printf("Cuda Producer: cuda Malloc failed, status:%d\n", status);
+    goto done;
+  }
+
+  atexit(presentApiStat);
+done:
+  return status;
+}
+
+CUresult cudaProducerDeinit(test_cuda_producer_s *cudaProducer) {
+  if (cudaProducer->tempBuff) {
+    free(cudaProducer->tempBuff);
+  }
+  if (cudaProducer->cudaPtr) {
+    cuMemFree(cudaProducer->cudaPtr);
+  }
+  return cuEGLStreamProducerDisconnect(&cudaProducer->cudaConn);
+}

Samples/EGLStream_CUDA_CrossGPU/cuda_producer.h

@@ -0,0 +1,68 @@
+/* Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+//
+// DESCRIPTION:   Simple cuda producer header file
+//
+
+#ifndef _CUDA_PRODUCER_H_
+#define _CUDA_PRODUCER_H_
+#include <EGL/egl.h>
+#include <EGL/eglext.h>
+#include "cudaEGL.h"
+#include "eglstrm_common.h"
+#include <cuda_runtime.h>
+#include <cuda.h>
+
+typedef struct _test_cuda_producer_s {
+  //  Stream params
+  CUcontext context;
+  CUeglStreamConnection cudaConn;
+  int cudaDevId;
+  EGLStreamKHR eglStream;
+  EGLDisplay eglDisplay;
+  unsigned int charCnt;
+  bool profileAPI;
+  char *tempBuff;
+  CUdeviceptr cudaPtr;
+  CUdeviceptr cudaPtr1;
+  CUstream prodCudaStream;
+} test_cuda_producer_s;
+
+CUresult cudaProducerInit(test_cuda_producer_s *cudaProducer, TestArgs *args);
+CUresult cudaProducerPresentFrame(test_cuda_producer_s *parserArg,
+                                  CUeglFrame cudaEgl, int t);
+CUresult cudaProducerReturnFrame(test_cuda_producer_s *parserArg,
+                                 CUeglFrame cudaEgl, int t);
+CUresult cudaProducerDeinit(test_cuda_producer_s *cudaProducer);
+CUresult cudaDeviceCreateProducer(test_cuda_producer_s *cudaProducer);
+cudaError_t cudaProducer_filter(CUstream cStream, char *pSrc, int width,
+                                int height, char expectedVal, char newVal,
+                                int frameNumber);
+void cudaProducerPrepareFrame(CUeglFrame *cudaEgl, CUdeviceptr cudaPtr,
+                              int bufferSize);
+#endif

Samples/EGLStream_CUDA_CrossGPU/eglstrm_common.cpp

@@ -0,0 +1,423 @@
+/* Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+//
+// DESCRIPTION:   Common egl stream functions
+//
+
+#include "eglstrm_common.h"
+
+EGLStreamKHR g_producerEglStream = EGL_NO_STREAM_KHR;
+EGLStreamKHR g_consumerEglStream = EGL_NO_STREAM_KHR;
+EGLDisplay g_producerEglDisplay = EGL_NO_DISPLAY;
+EGLDisplay g_consumerEglDisplay = EGL_NO_DISPLAY;
+int cudaDevIndexProd = -1;
+int cudaDevIndexCons = -1;
+
+#if defined(EXTENSION_LIST)
+EXTENSION_LIST(EXTLST_DECL)
+typedef void (*extlst_fnptr_t)(void);
+static struct {
+  extlst_fnptr_t *fnptr;
+  char const *name;
+  bool is_dgpu;  // This function is need only for dgpu case
+} extensionList[] = {EXTENSION_LIST(EXTLST_ENTRY)};
+
+int eglSetupExtensions(bool isCrossDevice) {
+  unsigned int i;
+
+  for (i = 0; i < (sizeof(extensionList) / sizeof(*extensionList)); i++) {
+    // load the dgpu function only if we are running cross device test
+    if ((!extensionList[i].is_dgpu) ||
+        (extensionList[i].is_dgpu == isCrossDevice)) {
+      *extensionList[i].fnptr = eglGetProcAddress(extensionList[i].name);
+      if (*extensionList[i].fnptr == NULL) {
+        printf("Couldn't get address of %s()\n", extensionList[i].name);
+        return 0;
+      }
+    }
+  }
+
+  return 1;
+}
+
+int EGLStreamInit(bool isCrossDevice, int isConsumer,
+                  EGLNativeFileDescriptorKHR fileDesc) {
+  static const EGLint streamAttrFIFOMode[] = {
+      EGL_STREAM_FIFO_LENGTH_KHR, 5, EGL_SUPPORT_REUSE_NV, EGL_FALSE, EGL_NONE};
+  EGLDisplay eglDisplay[2] = {0};
+  EGLStreamKHR eglStream[2] = {0};
+  EGLBoolean eglStatus;
+
+#define MAX_EGL_DEVICES 4
+
+  EGLDeviceEXT devices[MAX_EGL_DEVICES];
+  EGLint numDevices = 0;
+
+  eglStatus = eglQueryDevicesEXT(MAX_EGL_DEVICES, devices, &numDevices);
+  if (eglStatus != EGL_TRUE) {
+    printf("Error querying EGL devices\n");
+    goto Done;
+  }
+
+  if (numDevices == 0) {
+    printf("No EGL devices found\n");
+    eglStatus = EGL_FALSE;
+    goto Done;
+  }
+
+  // If cross device, create discrete GPU stream first and then create the
+  // integrated GPU stream to connect to it via fd. The other way round fails
+  // in producer connect.
+  //
+  // TODO: Find out if this EGL behavior is by design.
+  if (isConsumer) {
+    int egl_device_id = 0;
+    for (egl_device_id = 0; egl_device_id < numDevices; egl_device_id++) {
+      EGLAttrib cuda_device;
+      eglStatus = eglQueryDeviceAttribEXT(devices[egl_device_id],
+                                          EGL_CUDA_DEVICE_NV, &cuda_device);
+      if (eglStatus == EGL_TRUE) {
+        cudaDevIndexCons = cuda_device;  // We select first EGL-CUDA Capable
+                                         // device for consumer.
+        printf(
+            "Found EGL-CUDA Capable device with CUDA Device id = %d out of "
+            "egl_device_id = %d\n",
+            (int)cudaDevIndexCons, egl_device_id);
+        break;
+      }
+    }
+
+    if (egl_device_id >= numDevices) {
+      printf("No CUDA Capable EGL Device found.. Waiving execution\n");
+      goto Done;
+    }
+
+    g_consumerEglDisplay = eglGetPlatformDisplayEXT(
+        EGL_PLATFORM_DEVICE_EXT, (void *)devices[egl_device_id], NULL);
+    if (g_consumerEglDisplay == EGL_NO_DISPLAY) {
+      printf("Could not get EGL display from device. \n");
+      eglStatus = EGL_FALSE;
+      goto Done;
+    }
+
+    eglStatus = eglInitialize(g_consumerEglDisplay, 0, 0);
+    if (!eglStatus) {
+      printf("EGL failed to initialize. \n");
+      eglStatus = EGL_FALSE;
+      goto Done;
+    }
+
+    g_consumerEglStream =
+        eglCreateStreamKHR(g_consumerEglDisplay, streamAttrFIFOMode);
+    if (g_consumerEglStream == EGL_NO_STREAM_KHR) {
+      printf("Could not create EGL stream.\n");
+      eglStatus = EGL_FALSE;
+      goto Done;
+    }
+
+    eglStatus = eglStreamAttribKHR(g_consumerEglDisplay, g_consumerEglStream,
+                                   EGL_CONSUMER_LATENCY_USEC_KHR, 16000);
+    if (eglStatus != EGL_TRUE) {
+      printf("eglStreamAttribKHR EGL_CONSUMER_LATENCY_USEC_KHR failed\n");
+      goto Done;
+    }
+
+    eglStatus =
+        eglStreamAttribKHR(g_consumerEglDisplay, g_consumerEglStream,
+                           EGL_CONSUMER_ACQUIRE_TIMEOUT_USEC_KHR, 16000);
+    if (eglStatus != EGL_TRUE) {
+      printf(
+          "eglStreamAttribKHR EGL_CONSUMER_ACQUIRE_TIMEOUT_USEC_KHR failed\n");
+      goto Done;
+    }
+  }
+
+  if (!isConsumer) {  // Producer
+
+    if (fileDesc == EGL_NO_FILE_DESCRIPTOR_KHR) {
+      printf("Cuda Producer received bad file descriptor\n");
+      eglStatus = EGL_FALSE;
+      goto Done;
+    }
+
+    int egl_device_id = 0;
+    int egl_cuda_devices = 0;
+    for (egl_device_id = 0; egl_device_id < numDevices; egl_device_id++) {
+      EGLAttrib cuda_device = -1;
+      eglStatus = eglQueryDeviceAttribEXT(devices[egl_device_id],
+                                          EGL_CUDA_DEVICE_NV, &cuda_device);
+      if (eglStatus == EGL_TRUE) {
+        egl_cuda_devices++;
+        if (isCrossDevice && (egl_cuda_devices > 1)) {
+          // We select second EGL-CUDA Capable device for producer.
+          cudaDevIndexProd = (int)cuda_device;
+          printf(
+              "Found EGL-CUDA Capable device with CUDA Device id = %d "
+              "egl_device_id = %d \n",
+              (int)cudaDevIndexProd, egl_device_id);
+          break;
+        }
+        if (!isCrossDevice) {
+          // We select first EGL-CUDA Capable device for producer same as
+          // consumer.
+          cudaDevIndexProd = (int)cuda_device;
+          printf(
+              "Found EGL-CUDA Capable device with CUDA Device id = %d "
+              "egl_device_id = %d \n",
+              (int)cudaDevIndexProd, egl_device_id);
+          break;
+        }
+      }
+    }
+
+    if (egl_device_id >= numDevices) {
+      printf("No CUDA Capable EGL Device found.. Waiving execution\n");
+      goto Done;
+    }
+
+    g_producerEglDisplay = eglGetPlatformDisplayEXT(
+        EGL_PLATFORM_DEVICE_EXT, (void *)devices[egl_device_id], NULL);
+    if (g_producerEglDisplay == EGL_NO_DISPLAY) {
+      printf("Could not get EGL display from device. \n");
+      eglStatus = EGL_FALSE;
+      goto Done;
+    }
+
+    eglStatus = eglInitialize(g_producerEglDisplay, 0, 0);
+    if (!eglStatus) {
+      printf("EGL failed to initialize. \n");
+      eglStatus = EGL_FALSE;
+      goto Done;
+    }
+
+    g_producerEglStream =
+        eglCreateStreamFromFileDescriptorKHR(g_producerEglDisplay, fileDesc);
+    close(fileDesc);
+
+    if (g_producerEglStream == EGL_NO_STREAM_KHR) {
+      printf("CUDA Producer Could not create EGL stream.\n");
+      eglStatus = EGL_FALSE;
+      goto Done;
+    } else {
+      printf("Producer created EGLStream for the GPU.\n");
+    }
+  }
+
+Done:
+  return eglStatus == EGL_TRUE ? 1 : 0;
+}
+
+void EGLStreamFini(void) {
+  if (g_producerEglStream != EGL_NO_STREAM_KHR) {
+    eglDestroyStreamKHR(g_producerEglDisplay, g_producerEglStream);
+  }
+  if (g_consumerEglStream != g_producerEglStream) {
+    if (g_consumerEglStream != EGL_NO_STREAM_KHR) {
+      eglDestroyStreamKHR(g_consumerEglDisplay, g_consumerEglStream);
+    }
+  }
+}
+
+int UnixSocketConnect(const char *socket_name) {
+  int sock_fd = -1;
+  struct sockaddr_un sock_addr;
+  int wait_loop = 0;
+
+  sock_fd = socket(PF_UNIX, SOCK_STREAM, 0);
+  if (sock_fd < 0) {
+    printf("%s: socket create failed.\n", __func__);
+    return -1;
+  }
+
+  if (verbose) printf("%s: send_fd: sock_fd: %d\n", __func__, sock_fd);
+
+  memset(&sock_addr, 0, sizeof(struct sockaddr_un));
+  sock_addr.sun_family = AF_UNIX;
+  strncpy(sock_addr.sun_path, socket_name, sizeof(sock_addr.sun_path) - 1);
+
+  while (connect(sock_fd, (const struct sockaddr *)&sock_addr,
+                 sizeof(struct sockaddr_un))) {
+    if (wait_loop < 60) {
+      if (!wait_loop)
+        printf("Waiting for EGL stream producer ");
+      else
+        printf(".");
+      fflush(stdout);
+      sleep(1);
+      wait_loop++;
+    } else {
+      printf("\n%s: Waiting timed out\n", __func__);
+      return -1;
+    }
+  }
+  if (wait_loop) printf("\n");
+
+  if (verbose) printf("%s: Wait is done\n", __func__);
+
+  return sock_fd;
+}
+
+/* Send <fd_to_send> (a file descriptor) to another process */
+/* over a unix domain socket named <socket_name>.           */
+/* <socket_name> can be any nonexistant filename.           */
+int EGLStreamSendfd(int send_fd, int fd_to_send) {
+  struct msghdr msg;
+  struct iovec iov[1];
+  char ctrl_buf[CMSG_SPACE(sizeof(int))];
+  struct cmsghdr *cmsg = NULL;
+  void *data;
+  int res;
+  memset(&msg, 0, sizeof(msg));
+
+  iov[0].iov_len = 1;             // must send at least 1 byte
+  iov[0].iov_base = (void *)"x";  // any byte value (value ignored)
+  msg.msg_iov = iov;
+  msg.msg_iovlen = 1;
+
+  memset(ctrl_buf, 0, sizeof(ctrl_buf));
+  msg.msg_control = ctrl_buf;
+  msg.msg_controllen = sizeof(ctrl_buf);
+
+  cmsg = CMSG_FIRSTHDR(&msg);
+  cmsg->cmsg_level = SOL_SOCKET;
+  cmsg->cmsg_type = SCM_RIGHTS;
+  cmsg->cmsg_len = CMSG_LEN(sizeof(int));
+  data = CMSG_DATA(cmsg);
+  *(int *)data = fd_to_send;
+
+  msg.msg_controllen = cmsg->cmsg_len;
+
+  res = sendmsg(send_fd, &msg, 0);
+  if (res <= 0) {
+    printf("%s: sendmsg failed", __func__);
+    return -1;
+  }
+
+  return 0;
+}
+
+/* Listen on a unix domain socket named <socket_name>.     */
+/* Connect to it and return connect_fd                     */
+int UnixSocketCreate(const char *socket_name) {
+  int listen_fd;
+  struct sockaddr_un sock_addr;
+  int connect_fd;
+  struct sockaddr_un connect_addr;
+  socklen_t connect_addr_len = 0;
+
+  listen_fd = socket(PF_UNIX, SOCK_STREAM, 0);
+  if (listen_fd < 0) {
+    printf("%s: socket create failed", __func__);
+    return -1;
+  }
+
+  if (verbose) printf("%s: listen_fd: %d\n", __func__, listen_fd);
+
+  unlink(socket_name);
+
+  memset(&sock_addr, 0, sizeof(struct sockaddr_un));
+  sock_addr.sun_family = AF_UNIX;
+  strncpy(sock_addr.sun_path, socket_name, sizeof(sock_addr.sun_path) - 1);
+
+  if (bind(listen_fd, (const struct sockaddr *)&sock_addr,
+           sizeof(struct sockaddr_un))) {
+    printf("i%s: bind error", __func__);
+    return -1;
+  }
+
+  if (listen(listen_fd, 1)) {
+    printf("%s: listen error", __func__);
+    return -1;
+  }
+
+  connect_fd =
+      accept(listen_fd, (struct sockaddr *)&connect_addr, &connect_addr_len);
+
+  if (verbose) printf("%s: connect_fd: %d\n", __func__, connect_fd);
+
+  close(listen_fd);
+  unlink(socket_name);
+  if (connect_fd < 0) {
+    printf("%s: accept failed\n", __func__);
+    return -1;
+  }
+
+  return connect_fd;
+}
+
+/* receive a file descriptor from another process.         */
+/* Returns the file descriptor.  Note: the integer value   */
+/* of the file descriptor may be different from the        */
+/* integer value in the other process, but the file        */
+/* descriptors in each process will refer to the same file */
+/* object in the kernel.                                   */
+int EGLStreamReceivefd(int connect_fd) {
+  struct msghdr msg;
+  struct iovec iov[1];
+  char msg_buf[1];
+  char ctrl_buf[CMSG_SPACE(sizeof(int))];
+  struct cmsghdr *cmsg;
+  void *data;
+  int recvfd;
+
+  memset(&msg, 0, sizeof(msg));
+
+  iov[0].iov_base = msg_buf;
+  iov[0].iov_len = sizeof(msg_buf);
+  msg.msg_iov = iov;
+  msg.msg_iovlen = 1;
+
+  msg.msg_control = ctrl_buf;
+  msg.msg_controllen = sizeof(ctrl_buf);
+
+  if (recvmsg(connect_fd, &msg, 0) <= 0) {
+    printf("%s: recvmsg failed", __func__);
+    return -1;
+  }
+
+  cmsg = CMSG_FIRSTHDR(&msg);
+  if (!cmsg) {
+    printf("%s: NULL message header\n", __func__);
+    return -1;
+  }
+  if (cmsg->cmsg_level != SOL_SOCKET) {
+    printf("%s: Message level is not SOL_SOCKET\n", __func__);
+    return -1;
+  }
+  if (cmsg->cmsg_type != SCM_RIGHTS) {
+    printf("%s: Message type is not SCM_RIGHTS\n", __func__);
+    return -1;
+  }
+
+  data = CMSG_DATA(cmsg);
+  recvfd = *(int *)data;
+
+  return recvfd;
+}
+
+#endif

Samples/EGLStream_CUDA_CrossGPU/eglstrm_common.h

@@ -0,0 +1,113 @@
+/* Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+//
+// DESCRIPTION:   Common EGL stream functions header file
+//
+
+#ifndef _EGLSTRM_COMMON_H_
+#define _EGLSTRM_COMMON_H_
+
+#include <signal.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/socket.h>
+#include <sys/stat.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <sys/un.h>
+#include <time.h>
+#include <unistd.h>
+
+#include "cuda.h"
+#include "cudaEGL.h"
+#define TIME_DIFF(end, start) (getMicrosecond(end) - getMicrosecond(start))
+
+extern EGLStreamKHR g_producerEglStream;
+extern EGLStreamKHR g_consumerEglStream;
+extern EGLDisplay g_producerEglDisplay;
+extern EGLDisplay g_consumerEglDisplay;
+extern int cudaDevIndexCons;
+extern int cudaDevIndexProd;
+extern bool verbose;
+
+#define EXTENSION_LIST(T)                                                \
+  T(PFNEGLCREATESTREAMKHRPROC, eglCreateStreamKHR)                       \
+  T(PFNEGLDESTROYSTREAMKHRPROC, eglDestroyStreamKHR)                     \
+  T(PFNEGLQUERYSTREAMKHRPROC, eglQueryStreamKHR)                         \
+  T(PFNEGLQUERYSTREAMU64KHRPROC, eglQueryStreamu64KHR)                   \
+  T(PFNEGLQUERYSTREAMTIMEKHRPROC, eglQueryStreamTimeKHR)                 \
+  T(PFNEGLSTREAMATTRIBKHRPROC, eglStreamAttribKHR)                       \
+  T(PFNEGLSTREAMCONSUMERACQUIREKHRPROC, eglStreamConsumerAcquireKHR)     \
+  T(PFNEGLSTREAMCONSUMERRELEASEKHRPROC, eglStreamConsumerReleaseKHR)     \
+  T(PFNEGLSTREAMCONSUMERGLTEXTUREEXTERNALKHRPROC,                        \
+    eglStreamConsumerGLTextureExternalKHR)                               \
+  T(PFNEGLQUERYDEVICESEXTPROC, eglQueryDevicesEXT)                       \
+  T(PFNEGLGETPLATFORMDISPLAYEXTPROC, eglGetPlatformDisplayEXT)           \
+  T(PFNEGLGETSTREAMFILEDESCRIPTORKHRPROC, eglGetStreamFileDescriptorKHR) \
+  T(PFNEGLQUERYDEVICEATTRIBEXTPROC, eglQueryDeviceAttribEXT)             \
+  T(PFNEGLCREATESTREAMFROMFILEDESCRIPTORKHRPROC,                         \
+    eglCreateStreamFromFileDescriptorKHR)
+
+#define EXTLST_DECL(tx, x) tx x = NULL;
+#define EXTLST_EXTERN(tx, x) extern tx x;
+#define EXTLST_ENTRY(tx, x) {(extlst_fnptr_t *)&x, #x},
+
+#define MAX_STRING_SIZE 256
+#define INIT_DATA 0x01
+#define PROD_DATA 0x07
+#define CONS_DATA 0x04
+
+#define SOCK_PATH "/tmp/tegra_sw_egl_socket"
+
+typedef struct _TestArgs {
+  unsigned int charCnt;
+  bool isProducer;
+} TestArgs;
+
+extern int WIDTH, HEIGHT;
+
+int eglSetupExtensions(bool is_dgpu);
+int EGLStreamInit(bool isCrossDevice, int isConsumer,
+                  EGLNativeFileDescriptorKHR fileDesc);
+void EGLStreamFini(void);
+
+int EGLStreamSetAttr(EGLDisplay display, EGLStreamKHR eglStream);
+int UnixSocketConnect(const char *socket_name);
+int EGLStreamSendfd(int send_fd, int fd_to_send);
+int UnixSocketCreate(const char *socket_name);
+int EGLStreamReceivefd(int connect_fd);
+
+static clockid_t clock_id = CLOCK_MONOTONIC;  // CLOCK_PROCESS_CPUTIME_ID;
+static double getMicrosecond(struct timespec t) {
+  return ((t.tv_sec) * 1000000.0 + (t.tv_nsec) / 1.0e3);
+}
+
+static inline void getTime(struct timespec *t) { clock_gettime(clock_id, t); }
+#endif

Samples/EGLStream_CUDA_CrossGPU/findegl.mk

@@ -0,0 +1,160 @@
+################################################################################
+# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#  * Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#  * Neither the name of NVIDIA CORPORATION nor the names of its
+#    contributors may be used to endorse or promote products derived
+#    from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+################################################################################
+#
+#  findegl.mk is used to find the necessary EGL Libraries for specific distributions
+#               this is supported on Linux
+#
+################################################################################
+
+# Determine OS platform and unix distribution
+ifeq ("$(TARGET_OS)","linux")
+   # first search lsb_release
+   DISTRO  = $(shell lsb_release -i -s 2>/dev/null | tr "[:upper:]" "[:lower:]")
+   ifeq ("$(DISTRO)","")
+     # second search and parse /etc/issue
+     DISTRO = $(shell more /etc/issue | awk '{print $$1}' | sed '1!d' | sed -e "/^$$/d" 2>/dev/null | tr "[:upper:]" "[:lower:]")
+     # ensure data from /etc/issue is valid
+     ifeq (,$(filter $(DISTRO),ubuntu fedora red rhel centos suse))
+       DISTRO = 
+     endif
+     ifeq ("$(DISTRO)","")
+       # third, we can search in /etc/os-release or /etc/{distro}-release
+       DISTRO = $(shell awk '/ID/' /etc/*-release | sed 's/ID=//' | grep -v "VERSION" | grep -v "ID" | grep -v "DISTRIB")
+     endif
+   endif
+endif
+
+ifeq ("$(TARGET_OS)","linux")
+    # $(info) >> findegl.mk -> LINUX path <<<)
+    # Each set of Linux Distros have different paths for where to find their OpenGL libraries reside
+    UBUNTU = $(shell echo $(DISTRO) | grep -i ubuntu      >/dev/null 2>&1; echo $$?)
+    FEDORA = $(shell echo $(DISTRO) | grep -i fedora      >/dev/null 2>&1; echo $$?)
+    RHEL   = $(shell echo $(DISTRO) | grep -i 'red\|rhel' >/dev/null 2>&1; echo $$?)
+    CENTOS = $(shell echo $(DISTRO) | grep -i centos      >/dev/null 2>&1; echo $$?)
+    SUSE   = $(shell echo $(DISTRO) | grep -i 'suse\|sles' >/dev/null 2>&1; echo $$?)
+    ifeq ("$(UBUNTU)","0")
+      ifeq ($(HOST_ARCH)-$(TARGET_ARCH),x86_64-armv7l)
+        GLPATH := /usr/arm-linux-gnueabihf/lib
+        GLLINK := -L/usr/arm-linux-gnueabihf/lib
+        ifneq ($(TARGET_FS),) 
+          GLPATH += $(TARGET_FS)/usr/lib/arm-linux-gnueabihf
+          GLLINK += -L$(TARGET_FS)/usr/lib/arm-linux-gnueabihf
+        endif
+      else ifeq ($(HOST_ARCH)-$(TARGET_ARCH),x86_64-aarch64)
+        GLPATH := /usr/aarch64-linux-gnu/lib
+        GLLINK := -L/usr/aarch64-linux-gnu/lib
+        ifneq ($(TARGET_FS),)
+          GLPATH += $(TARGET_FS)/usr/lib
+          GLPATH += $(TARGET_FS)/usr/lib/aarch64-linux-gnu
+          GLLINK += -L$(TARGET_FS)/usr/lib/aarch64-linux-gnu
+        endif 
+      else
+        UBUNTU_PKG_NAME = $(shell which dpkg >/dev/null 2>&1 && dpkg -l 'nvidia-*' | grep '^ii' | awk '{print $$2}' | head -1)
+        ifneq ("$(UBUNTU_PKG_NAME)","")
+          GLPATH    ?= /usr/lib/$(UBUNTU_PKG_NAME)
+          GLLINK    ?= -L/usr/lib/$(UBUNTU_PKG_NAME)
+        endif
+
+        DFLT_PATH ?= /usr/lib
+      endif
+    endif
+    ifeq ("$(SUSE)","0")
+      GLPATH    ?= /usr/X11R6/lib64
+      GLLINK    ?= -L/usr/X11R6/lib64
+      DFLT_PATH ?= /usr/lib64
+    endif
+    ifeq ("$(FEDORA)","0")
+      GLPATH    ?= /usr/lib64/nvidia
+      GLLINK    ?= -L/usr/lib64/nvidia
+      DFLT_PATH ?= /usr/lib64
+    endif
+    ifeq ("$(RHEL)","0")
+      GLPATH    ?= /usr/lib64/nvidia
+      GLLINK    ?= -L/usr/lib64/nvidia
+      DFLT_PATH ?= /usr/lib64
+    endif
+    ifeq ("$(CENTOS)","0")
+      GLPATH    ?= /usr/lib64/nvidia
+      GLLINK    ?= -L/usr/lib64/nvidia
+      DFLT_PATH ?= /usr/lib64
+    endif
+
+  EGLLIB  := $(shell find -L $(GLPATH) $(DFLT_PATH) -name libEGL.so    -print 2>/dev/null)
+
+  ifeq ("$(EGLLIB)","")
+      $(info >>> WARNING - libEGL.so not found, please install libEGL.so <<<)
+      SAMPLE_ENABLED := 0
+  endif
+
+  HEADER_SEARCH_PATH ?= $(TARGET_FS)/usr/include
+  ifeq ($(HOST_ARCH)-$(TARGET_ARCH)-$(TARGET_OS),x86_64-armv7l-linux)
+      HEADER_SEARCH_PATH += /usr/arm-linux-gnueabihf/include
+  else ifeq ($(HOST_ARCH)-$(TARGET_ARCH)-$(TARGET_OS),x86_64-aarch64-linux)
+      HEADER_SEARCH_PATH += /usr/aarch64-linux-gnu/include
+  endif
+
+  EGLHEADER  := $(shell find -L $(HEADER_SEARCH_PATH) -name egl.h -print 2>/dev/null)
+  EGLEXTHEADER  := $(shell find -L $(HEADER_SEARCH_PATH) -name eglext.h -print 2>/dev/null)
+
+  ifeq ("$(EGLHEADER)","")
+      $(info >>> WARNING - egl.h not found, please install egl.h <<<)
+      SAMPLE_ENABLED := 0
+  endif
+  ifeq ("$(EGLEXTHEADER)","")
+      $(info >>> WARNING - eglext.h not found, please install eglext.h <<<)
+      SAMPLE_ENABLED := 0
+  endif
+else
+endif
+
+ifeq ("$(TARGET_OS)","qnx")
+    HOST_CCFLAGS := -V5.4.0,gcc_ntoaarch64le
+endif
+
+# Attempt to compile a minimal EGL application and run to check if EGL_SUPPORT_REUSE_NV is supported in the EGL headers available.
+ifneq ($(SAMPLE_ENABLED), 0)
+      $(shell printf "#include <EGL/egl.h>\n#include <EGL/eglext.h>\nint main() {\n#ifdef EGL_SUPPORT_REUSE_NV \n #error \"Compatible EGL header found\" \n  return 0;\n#endif \n return 1;\n}"  > test.c; )
+      EGL_DEFINES := $(shell $(HOST_COMPILER) $(HOST_CCFLAGS) $(CCFLAGS) $(EXTRA_CCFLAGS) -lEGL test.c -c 2>&1 | grep -ic "Compatible EGL header found";)
+      SHOULD_WAIVE := 0
+      ifeq ($(EGL_DEFINES),0)
+        SHOULD_WAIVE := 1
+      endif
+      ifeq ($(SHOULD_WAIVE),1)
+          $(info -----------------------------------------------------------------------------------------------)
+          $(info WARNING - NVIDIA EGL EXTENSIONS are not available in the present EGL headers)
+          $(info -----------------------------------------------------------------------------------------------)
+          $(info   This CUDA Sample cannot be built if the EGL NVIDIA EXTENSIONS like EGL_SUPPORT_REUSE_NV are not supported in EGL headers.)
+          $(info   This will be a dry-run of the Makefile.)
+          $(info   Please install the latest khronos EGL headers and libs to build this sample)
+          $(info -----------------------------------------------------------------------------------------------)
+          SAMPLE_ENABLED := 0
+      endif
+      $(shell rm test.o test.c 2>/dev/null)
+endif
+

Samples/EGLStream_CUDA_CrossGPU/helper.h

@@ -0,0 +1,221 @@
+/* Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "eglstrm_common.h"
+#if defined(EXTENSION_LIST)
+EXTENSION_LIST(EXTLST_EXTERN)
+#endif
+#include <cuda.h>
+
+int parseCmdLine(int argc, char *argv[], TestArgs *args);
+void printUsage(void);
+int NUMTRIALS = 10;
+int profileAPIs = 0;
+
+bool verbose = 0;
+bool isCrossDevice = 0;
+
+// Parse the command line options. Returns FAILURE on a parse error, SUCCESS
+// otherwise.
+int parseCmdLine(int argc, char *argv[], TestArgs *args) {
+  int i;
+
+  for (i = 1; i < argc; i++) {
+    if (strcmp(argv[i], "-h") == 0) {
+      printUsage();
+      exit(0);
+    } else if (strcmp(argv[i], "-n") == 0) {
+      ++i;
+      if (sscanf(argv[i], "%d", &NUMTRIALS) != 1 || NUMTRIALS <= 0) {
+        printf("Invalid trial count: %s should be > 0\n", argv[i]);
+        return -1;
+      }
+    } else if (strcmp(argv[i], "-profile") == 0) {
+      profileAPIs = 1;
+    } else if (strcmp(argv[i], "-crossdev") == 0) {
+      isCrossDevice = 1;
+    } else if (strcmp(argv[i], "-width") == 0) {
+      ++i;
+      if (sscanf(argv[i], "%d", &WIDTH) != 1 || (WIDTH <= 0)) {
+        printf("Width should be greater than 0\n");
+        return -1;
+      }
+    } else if (strcmp(argv[i], "-height") == 0) {
+      ++i;
+      if (sscanf(argv[i], "%d", &HEIGHT) != 1 || (HEIGHT <= 0)) {
+        printf("Width should be greater than 0\n");
+        return -1;
+      }
+    } else if (0 == strcmp(&argv[i][1], "proctype")) {
+      ++i;
+      if (!strcasecmp(argv[i], "prod")) {
+        args->isProducer = 1;
+      } else if (!strcasecmp(argv[i], "cons")) {
+        args->isProducer = 0;
+      } else {
+        printf("%s: Bad Process Type: %s\n", __func__, argv[i]);
+        return 1;
+      }
+    } else if (strcmp(argv[i], "-v") == 0) {
+      verbose = 1;
+    } else {
+      printf("Unknown option: %s\n", argv[i]);
+      return -1;
+    }
+  }
+
+  if (isCrossDevice) {
+    int deviceCount = 0;
+
+    CUresult error_id = cuInit(0);
+    if (error_id != CUDA_SUCCESS) {
+      printf("cuInit(0) returned %d\n", error_id);
+      printf("Result = FAIL\n");
+      exit(EXIT_FAILURE);
+    }
+
+    error_id = cuDeviceGetCount(&deviceCount);
+    if (error_id != CUDA_SUCCESS) {
+      printf("cuDeviceGetCount returned %d\n", (int)error_id);
+      printf("Result = FAIL\n");
+      exit(EXIT_FAILURE);
+    }
+
+    int iGPUexists = 0;
+    CUdevice dev;
+    for (dev = 0; dev < deviceCount; ++dev) {
+      int integrated = 0;
+      CUresult error_result = cuDeviceGetAttribute(
+          &integrated, CU_DEVICE_ATTRIBUTE_INTEGRATED, dev);
+
+      if (error_result != CUDA_SUCCESS) {
+        printf("cuDeviceGetAttribute returned error : %d\n", (int)error_result);
+        exit(EXIT_FAILURE);
+      }
+
+      if (integrated) {
+        iGPUexists = 1;
+      }
+    }
+
+    if (!iGPUexists) {
+      printf("No Integrated GPU found in the system.\n");
+      printf(
+          "-crossdev option is only supported on systems with an Integrated "
+          "GPU and a Discrete GPU\n");
+      printf("Waiving the execution\n");
+      exit(EXIT_SUCCESS);
+    }
+  }
+
+  if (!eglSetupExtensions(isCrossDevice)) {
+    printf("SetupExtentions failed \n");
+    exit(EXIT_FAILURE);
+  }
+#define MAX_EGL_DEVICES 4
+  EGLDeviceEXT devices[MAX_EGL_DEVICES];
+  EGLint numDevices = 0;
+  EGLBoolean eglStatus =
+      eglQueryDevicesEXT(MAX_EGL_DEVICES, devices, &numDevices);
+  if (eglStatus != EGL_TRUE) {
+    printf("Error querying EGL devices\n");
+    exit(EXIT_FAILURE);
+  }
+
+  if (numDevices == 0) {
+    printf("No EGL devices found\n");
+    eglStatus = EGL_FALSE;
+    exit(2);  // EXIT_WAIVED
+  }
+
+  int egl_device_id = 0;
+  for (egl_device_id = 0; egl_device_id < numDevices; egl_device_id++) {
+    EGLAttrib cuda_device;
+    eglStatus = eglQueryDeviceAttribEXT(devices[egl_device_id],
+                                        EGL_CUDA_DEVICE_NV, &cuda_device);
+    if (eglStatus == EGL_TRUE) {
+      break;
+    }
+  }
+
+  if (egl_device_id >= numDevices) {
+    printf("No CUDA Capable EGL Device found.. Waiving execution\n");
+    exit(2);  // EXIT_WAIVED
+  }
+
+  if (isCrossDevice) {
+    if (numDevices == 1) {
+      printf(
+          "Found only one EGL device, cannot setup cross GPU streams. "
+          "Waiving\n");
+      eglStatus = EGL_FALSE;
+      exit(2);  // EXIT_WAIVED
+    }
+  }
+
+  return 0;
+}
+
+void launchProducer(TestArgs *args) {
+  /* Cross-process creation of producer */
+  char argsProducer[1024];
+  char str[256];
+
+  strcpy(argsProducer, "./EGLStream_CUDA_CrossGPU -proctype prod ");
+
+  if (isCrossDevice) {
+    sprintf(str, "-crossdev ");
+    strcat(argsProducer, str);
+  }
+
+  if (verbose) {
+    sprintf(str, "-v ");
+    strcat(argsProducer, str);
+  }
+
+  /*Make the process run in bg*/
+  strcat(argsProducer, "& ");
+
+  printf("\n%s: Crossproc Producer command: %s \n", __func__, argsProducer);
+
+  /*Create crossproc Producer*/
+  system(argsProducer);
+
+  /*Enable crossproc Consumer in the same process */
+  args->isProducer = 0;
+}
+
+void printUsage(void) {
+  printf("Usage:\n");
+  printf("  -h           Print this help message\n");
+  printf("  -n n         Exit after running n trials. Set to 10 by default\n");
+  printf(
+      "  -profile     Profile time taken by ReleaseAPI. Not set by default\n");
+  printf("  -crossdev    Run with producer on idgpu and consumer on dgpu\n");
+  printf("  -dgpu        (same as -crossdev, deprecated)\n");
+  printf("  -v           verbose output\n");
+}

Samples/EGLStream_CUDA_CrossGPU/kernel.cu

@@ -0,0 +1,140 @@
+/* Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+//
+// DESCRIPTION:   Simple CUDA consumer rendering sample app
+//
+
+#include <EGL/egl.h>
+#include <EGL/eglext.h>
+#include <cuda.h>
+#include <cuda_runtime.h>
+#include <stdio.h>
+#include <string.h>
+
+#include "eglstrm_common.h"
+
+extern bool isCrossDevice;
+
+__device__ static unsigned int numErrors = 0, errorFound = 0;
+__device__ void checkProducerDataGPU(char *data, int size, char expectedVal,
+                                     int frameNumber) {
+  if ((data[blockDim.x * blockIdx.x + threadIdx.x] != expectedVal) &&
+      (!errorFound)) {
+    printf("Producer FOUND:%d expected: %d at %d for trial %d %d\n",
+           data[blockDim.x * blockIdx.x + threadIdx.x], expectedVal,
+           (blockDim.x * blockIdx.x + threadIdx.x), frameNumber, numErrors);
+    numErrors++;
+    errorFound = 1;
+    return;
+  }
+}
+
+__device__ void checkConsumerDataGPU(char *data, int size, char expectedVal,
+                                     int frameNumber) {
+  if ((data[blockDim.x * blockIdx.x + threadIdx.x] != expectedVal) &&
+      (!errorFound)) {
+    printf("Consumer FOUND:%d expected: %d at %d for trial %d %d\n",
+           data[blockDim.x * blockIdx.x + threadIdx.x], expectedVal,
+           (blockDim.x * blockIdx.x + threadIdx.x), frameNumber, numErrors);
+    numErrors++;
+    errorFound = 1;
+    return;
+  }
+}
+
+__global__ void writeDataToBuffer(char *pSrc, char newVal) {
+  pSrc[blockDim.x * blockIdx.x + threadIdx.x] = newVal;
+}
+
+__global__ void testKernelConsumer(char *pSrc, char size, char expectedVal,
+                                   char newVal, int frameNumber) {
+  checkConsumerDataGPU(pSrc, size, expectedVal, frameNumber);
+}
+
+__global__ void testKernelProducer(char *pSrc, char size, char expectedVal,
+                                   char newVal, int frameNumber) {
+  checkProducerDataGPU(pSrc, size, expectedVal, frameNumber);
+}
+__global__ void getNumErrors(int *numErr) { *numErr = numErrors; }
+
+cudaError_t cudaProducer_filter(cudaStream_t pStream, char *pSrc, int width,
+                                int height, char expectedVal, char newVal,
+                                int frameNumber) {
+  // in case where consumer is on dgpu and producer is on igpu when return is
+  // called the frame is not copied back to igpu. So the consumer changes is not
+  // visible to producer
+  if (isCrossDevice == 0) {
+    testKernelProducer<<<(width * height) / 1024, 1024, 1, pStream>>>(
+        pSrc, width * height, expectedVal, newVal, frameNumber);
+  }
+  writeDataToBuffer<<<(width * height) / 1024, 1024, 1, pStream>>>(pSrc,
+                                                                   newVal);
+  return cudaSuccess;
+};
+
+cudaError_t cudaConsumer_filter(cudaStream_t cStream, char *pSrc, int width,
+                                int height, char expectedVal, char newVal,
+                                int frameNumber) {
+  testKernelConsumer<<<(width * height) / 1024, 1024, 1, cStream>>>(
+      pSrc, width * height, expectedVal, newVal, frameNumber);
+  writeDataToBuffer<<<(width * height) / 1024, 1024, 1, cStream>>>(pSrc,
+                                                                   newVal);
+  return cudaSuccess;
+};
+
+cudaError_t cudaGetValueMismatch() {
+  int numErr_h;
+  int *numErr_d = NULL;
+  cudaError_t err = cudaSuccess;
+  err = cudaMalloc(&numErr_d, sizeof(int));
+  if (err != cudaSuccess) {
+    printf("Cuda Main: cudaMalloc failed with %s\n", cudaGetErrorString(err));
+    return err;
+  }
+  getNumErrors<<<1, 1>>>(numErr_d);
+  err = cudaDeviceSynchronize();
+  if (err != cudaSuccess) {
+    printf("Cuda Main: cudaDeviceSynchronize failed with %s\n",
+           cudaGetErrorString(err));
+  }
+  err = cudaMemcpy(&numErr_h, numErr_d, sizeof(int), cudaMemcpyDeviceToHost);
+  if (err != cudaSuccess) {
+    printf("Cuda Main: cudaMemcpy failed with %s\n", cudaGetErrorString(err));
+    cudaFree(numErr_d);
+    return err;
+  }
+  err = cudaFree(numErr_d);
+  if (err != cudaSuccess) {
+    printf("Cuda Main: cudaFree failed with %s\n", cudaGetErrorString(err));
+    return err;
+  }
+  if (numErr_h > 0) {
+    return cudaErrorUnknown;
+  }
+  return cudaSuccess;
+}

Samples/EGLStream_CUDA_CrossGPU/main.cpp

@@ -0,0 +1,392 @@
+/* Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "cudaEGL.h"
+#include "cuda_consumer.h"
+#include "cuda_producer.h"
+#include "eglstrm_common.h"
+#include "helper.h"
+#if defined(EXTENSION_LIST)
+EXTENSION_LIST(EXTLST_EXTERN)
+#endif
+
+bool signal_stop = 0;
+extern bool verbose;
+
+static void sig_handler(int sig) {
+  signal_stop = 1;
+  printf("Signal: %d\n", sig);
+}
+
+void DoneCons(int consumerStatus, int send_fd) {
+  EGLStreamFini();
+  // get the final status from producer, combine and print
+  int producerStatus = -1;
+  if (-1 == recv(send_fd, (void *)&producerStatus, sizeof(int), 0)) {
+    printf("%s: Cuda Consumer could not receive status from producer.\n",
+           __func__);
+  }
+  close(send_fd);
+
+  if (producerStatus == 0 && consumerStatus == 0) {
+    printf("&&&& EGLStream_CUDA_CrossGPU PASSED\n");
+    exit(EXIT_SUCCESS);
+  } else {
+    printf("&&&& EGLStream_CUDA_CrossGPU FAILED\n");
+    exit(EXIT_FAILURE);
+  }
+}
+
+void DoneProd(int producerStatus, int connect_fd) {
+  EGLStreamFini();
+  if (-1 == send(connect_fd, (void *)&producerStatus, sizeof(int), 0)) {
+    printf("%s: Cuda Producer could not send status to consumer.\n", __func__);
+  }
+  close(connect_fd);
+  if (producerStatus == 0) {
+    exit(EXIT_SUCCESS);
+  } else {
+    exit(EXIT_FAILURE);
+  }
+}
+
+int WIDTH = 8192, HEIGHT = 8192;
+int main(int argc, char **argv) {
+  TestArgs args = {0, false};
+  CUresult curesult = CUDA_SUCCESS;
+  unsigned int j = 0;
+  cudaError_t err = cudaSuccess;
+  EGLNativeFileDescriptorKHR fileDescriptor = EGL_NO_FILE_DESCRIPTOR_KHR;
+  struct timespec start, end;
+  CUeglFrame cudaEgl1, cudaEgl2;
+  int consumerStatus = 0;
+  int send_fd = -1;
+
+  if (parseCmdLine(argc, argv, &args) < 0) {
+    printUsage();
+    curesult = CUDA_ERROR_UNKNOWN;
+    DoneCons(consumerStatus, send_fd);
+  }
+
+  printf("Width : %u, height: %u and iterations: %u\n", WIDTH, HEIGHT,
+         NUMTRIALS);
+
+  if (!args.isProducer)  // Consumer code
+  {
+    test_cuda_consumer_s cudaConsumer;
+    memset(&cudaConsumer, 0, sizeof(test_cuda_consumer_s));
+    cudaConsumer.profileAPI = profileAPIs;
+
+    // Hook up Ctrl-C handler
+    signal(SIGINT, sig_handler);
+
+    if (!EGLStreamInit(isCrossDevice, !args.isProducer,
+                       EGL_NO_FILE_DESCRIPTOR_KHR)) {
+      printf("EGLStream Init failed.\n");
+      curesult = CUDA_ERROR_UNKNOWN;
+      consumerStatus = -1;
+      DoneCons(consumerStatus, send_fd);
+    }
+
+    cudaConsumer.cudaDevId = cudaDevIndexCons;
+    curesult = cudaDeviceCreateConsumer(&cudaConsumer);
+    if (curesult != CUDA_SUCCESS) {
+      consumerStatus = -1;
+      DoneCons(consumerStatus, send_fd);
+    }
+
+    cuCtxPushCurrent(cudaConsumer.context);
+
+    launchProducer(&args);
+
+    args.charCnt = WIDTH * HEIGHT * 4;
+
+    curesult = cuda_consumer_init(&cudaConsumer, &args);
+    if (curesult != CUDA_SUCCESS) {
+      printf("Cuda Consumer: Init failed, status: %d\n", curesult);
+      consumerStatus = -1;
+      DoneCons(consumerStatus, send_fd);
+    }
+
+    cuCtxPopCurrent(&cudaConsumer.context);
+
+    send_fd = UnixSocketConnect(SOCK_PATH);
+    if (-1 == send_fd) {
+      printf("%s: Cuda Consumer cannot create socket %s\n", __func__,
+             SOCK_PATH);
+      consumerStatus = -1;
+      DoneCons(consumerStatus, send_fd);
+    }
+
+    cuCtxPushCurrent(cudaConsumer.context);
+    cudaConsumer.eglStream = g_consumerEglStream;
+    cudaConsumer.eglDisplay = g_consumerEglDisplay;
+
+    // Send the EGL stream FD to producer
+    fileDescriptor = eglGetStreamFileDescriptorKHR(cudaConsumer.eglDisplay,
+                                                   cudaConsumer.eglStream);
+    if (EGL_NO_FILE_DESCRIPTOR_KHR == fileDescriptor) {
+      printf("%s: Cuda Consumer could not get EGL file descriptor.\n",
+             __func__);
+      eglDestroyStreamKHR(cudaConsumer.eglDisplay, cudaConsumer.eglStream);
+      consumerStatus = -1;
+      DoneCons(consumerStatus, send_fd);
+    }
+
+    if (verbose)
+      printf("%s: Cuda Consumer EGL stream FD obtained : %d.\n", __func__,
+             fileDescriptor);
+
+    int res = -1;
+    res = EGLStreamSendfd(send_fd, fileDescriptor);
+    if (-1 == res) {
+      printf("%s: Cuda Consumer could not send EGL file descriptor.\n",
+             __func__);
+      consumerStatus = -1;
+      close(fileDescriptor);
+    }
+
+    if (CUDA_SUCCESS !=
+        (curesult = cuEGLStreamConsumerConnect(&(cudaConsumer.cudaConn),
+                                               cudaConsumer.eglStream))) {
+      printf("FAILED Connect CUDA consumer with error %d\n", curesult);
+      consumerStatus = -1;
+      DoneCons(consumerStatus, send_fd);
+    }
+
+    j = 0;
+    for (j = 0; j < NUMTRIALS; j++) {
+      curesult = cudaConsumerAcquireFrame(&cudaConsumer, j);
+      if (curesult != CUDA_SUCCESS) {
+        printf("Cuda Consumer Test failed for frame = %d\n", j + 1);
+        consumerStatus = -1;
+        DoneCons(consumerStatus, send_fd);
+      }
+      curesult = cudaConsumerReleaseFrame(&cudaConsumer, j);
+      if (curesult != CUDA_SUCCESS) {
+        printf("Cuda Consumer Test failed for frame = %d\n", j + 1);
+        consumerStatus = -1;
+        DoneCons(consumerStatus, send_fd);
+      }
+
+      curesult = cudaConsumerAcquireFrame(&cudaConsumer, j);
+      if (curesult != CUDA_SUCCESS) {
+        printf("Cuda Consumer Test failed for frame = %d\n", j + 1);
+        consumerStatus = -1;
+        DoneCons(consumerStatus, send_fd);
+      }
+      curesult = cudaConsumerReleaseFrame(&cudaConsumer, j);
+      if (curesult != CUDA_SUCCESS) {
+        printf("Cuda Consumer Test failed for frame = %d\n", j + 1);
+        consumerStatus = -1;
+        DoneCons(consumerStatus, send_fd);
+      }
+    }
+    cuCtxSynchronize();
+    close(fileDescriptor);
+    err = cudaGetValueMismatch();
+    if (err != cudaSuccess) {
+      printf("Consumer: App failed with value mismatch\n");
+      curesult = CUDA_ERROR_UNKNOWN;
+      consumerStatus = -1;
+      DoneCons(consumerStatus, send_fd);
+    }
+
+    EGLint streamState = 0;
+    if (!eglQueryStreamKHR(cudaConsumer.eglDisplay, cudaConsumer.eglStream,
+                           EGL_STREAM_STATE_KHR, &streamState)) {
+      printf("Main, eglQueryStreamKHR EGL_STREAM_STATE_KHR failed\n");
+      curesult = CUDA_ERROR_UNKNOWN;
+      consumerStatus = -1;
+      DoneCons(consumerStatus, send_fd);
+    }
+
+    if (streamState != EGL_STREAM_STATE_DISCONNECTED_KHR) {
+      if (CUDA_SUCCESS != (curesult = cuda_consumer_Deinit(&cudaConsumer))) {
+        printf("Consumer Disconnect FAILED.\n");
+        consumerStatus = -1;
+        DoneCons(consumerStatus, send_fd);
+      }
+    }
+  } else  // Producer
+  {
+    test_cuda_producer_s cudaProducer;
+    memset(&cudaProducer, 0, sizeof(test_cuda_producer_s));
+    cudaProducer.profileAPI = profileAPIs;
+    int producerStatus = 0;
+
+    setenv("CUDA_EGL_PRODUCER_RETURN_WAIT_TIMEOUT", "1600", 0);
+
+    int connect_fd = -1;
+    // Hook up Ctrl-C handler
+    signal(SIGINT, sig_handler);
+
+    // Create connection to Consumer
+    connect_fd = UnixSocketCreate(SOCK_PATH);
+    if (-1 == connect_fd) {
+      printf("%s: Cuda Producer could not create socket: %s.\n", __func__,
+             SOCK_PATH);
+      producerStatus = -1;
+      DoneProd(producerStatus, connect_fd);
+    }
+
+    // Get the file descriptor of the stream from the consumer process
+    // and re-create the EGL stream from it
+    fileDescriptor = EGLStreamReceivefd(connect_fd);
+    if (-1 == fileDescriptor) {
+      printf("%s: Cuda Producer could not receive EGL file descriptor \n",
+             __func__);
+      producerStatus = -1;
+      DoneProd(producerStatus, connect_fd);
+    }
+
+    if (!EGLStreamInit(isCrossDevice, 0, fileDescriptor)) {
+      printf("EGLStream Init failed.\n");
+      producerStatus = -1;
+      curesult = CUDA_ERROR_UNKNOWN;
+      DoneProd(producerStatus, connect_fd);
+    }
+
+    cudaProducer.eglDisplay = g_producerEglDisplay;
+    cudaProducer.eglStream = g_producerEglStream;
+    cudaProducer.cudaDevId = cudaDevIndexProd;
+
+    curesult = cudaDeviceCreateProducer(&cudaProducer);
+    if (curesult != CUDA_SUCCESS) {
+      producerStatus = -1;
+      DoneProd(producerStatus, connect_fd);
+    }
+
+    args.charCnt = WIDTH * HEIGHT * 4;
+    cuCtxPushCurrent(cudaProducer.context);
+    curesult = cudaProducerInit(&cudaProducer, &args);
+    if (curesult != CUDA_SUCCESS) {
+      printf("Cuda Producer: Init failed, status: %d\n", curesult);
+      producerStatus = -1;
+      DoneProd(producerStatus, connect_fd);
+    }
+
+    // wait for consumer to connect first
+    int err = 0;
+    int wait_loop = 0;
+    EGLint streamState = 0;
+    do {
+      err = eglQueryStreamKHR(cudaProducer.eglDisplay, cudaProducer.eglStream,
+                              EGL_STREAM_STATE_KHR, &streamState);
+      if ((0 != err) && (EGL_STREAM_STATE_CONNECTING_KHR != streamState)) {
+        sleep(1);
+        wait_loop++;
+      }
+    } while ((wait_loop < 10) && (0 != err) &&
+             (streamState != EGL_STREAM_STATE_CONNECTING_KHR));
+
+    if ((0 == err) || (wait_loop >= 10)) {
+      printf(
+          "%s: Cuda Producer eglQueryStreamKHR EGL_STREAM_STATE_KHR failed.\n",
+          __func__);
+      producerStatus = -1;
+      DoneProd(producerStatus, connect_fd);
+    }
+
+    if (CUDA_SUCCESS != (curesult = cuEGLStreamProducerConnect(
+                             &(cudaProducer.cudaConn), cudaProducer.eglStream,
+                             WIDTH, HEIGHT))) {
+      printf("Connect CUDA producer FAILED with error %d\n", curesult);
+      producerStatus = -1;
+      DoneProd(producerStatus, connect_fd);
+    }
+
+    printf("main - Cuda Producer and Consumer Initialized.\n");
+
+    cudaProducerPrepareFrame(&cudaEgl1, cudaProducer.cudaPtr, args.charCnt);
+    cudaProducerPrepareFrame(&cudaEgl2, cudaProducer.cudaPtr1, args.charCnt);
+
+    j = 0;
+    for (j = 0; j < NUMTRIALS; j++) {
+      curesult = cudaProducerPresentFrame(&cudaProducer, cudaEgl1, j);
+      if (curesult != CUDA_SUCCESS) {
+        printf("Cuda Producer Test failed for frame = %d with cuda error:%d\n",
+               j + 1, curesult);
+        producerStatus = -1;
+        DoneProd(producerStatus, connect_fd);
+      }
+
+      curesult = cudaProducerPresentFrame(&cudaProducer, cudaEgl2, j);
+      if (curesult != CUDA_SUCCESS) {
+        printf("Cuda Producer Test failed for frame = %d with cuda error:%d\n",
+               j + 1, curesult);
+        producerStatus = -1;
+        DoneProd(producerStatus, connect_fd);
+      }
+
+      curesult = cudaProducerReturnFrame(&cudaProducer, cudaEgl1, j);
+      if (curesult != CUDA_SUCCESS) {
+        printf("Cuda Producer Test failed for frame = %d with cuda error:%d\n",
+               j + 1, curesult);
+        producerStatus = -1;
+        DoneProd(producerStatus, connect_fd);
+      }
+
+      curesult = cudaProducerReturnFrame(&cudaProducer, cudaEgl2, j);
+      if (curesult != CUDA_SUCCESS) {
+        printf("Cuda Producer Test failed for frame = %d with cuda error:%d\n",
+               j + 1, curesult);
+        producerStatus = -1;
+        DoneProd(producerStatus, connect_fd);
+      }
+    }
+
+    cuCtxSynchronize();
+    err = cudaGetValueMismatch();
+    if (err != cudaSuccess) {
+      printf("Prod: App failed with value mismatch\n");
+      curesult = CUDA_ERROR_UNKNOWN;
+      producerStatus = -1;
+      DoneProd(producerStatus, connect_fd);
+    }
+
+    printf("Tear Down Start.....\n");
+    if (!eglQueryStreamKHR(cudaProducer.eglDisplay, cudaProducer.eglStream,
+                           EGL_STREAM_STATE_KHR, &streamState)) {
+      printf("Main, eglQueryStreamKHR EGL_STREAM_STATE_KHR failed\n");
+      curesult = CUDA_ERROR_UNKNOWN;
+      producerStatus = -1;
+      DoneProd(producerStatus, connect_fd);
+    }
+
+    if (streamState != EGL_STREAM_STATE_DISCONNECTED_KHR) {
+      if (CUDA_SUCCESS != (curesult = cudaProducerDeinit(&cudaProducer))) {
+        printf("Producer Disconnect FAILED with %d\n", curesult);
+        producerStatus = -1;
+        DoneProd(producerStatus, connect_fd);
+      }
+    }
+    unsetenv("CUDA_EGL_PRODUCER_RETURN_WAIT_TIMEOUT");
+  }
+
+  return 0;
+}

Samples/EGLStreams_CUDA_Interop/.vscode/c_cpp_properties.json

@@ -0,0 +1,18 @@
+{
+    "configurations": [
+        {
+            "name": "Linux",
+            "includePath": [
+                "${workspaceFolder}/**",
+                "${workspaceFolder}/../../Common"
+            ],
+            "defines": [],
+            "compilerPath": "/usr/local/cuda/bin/nvcc",
+            "cStandard": "gnu17",
+            "cppStandard": "gnu++14",
+            "intelliSenseMode": "linux-gcc-x64",
+            "configurationProvider": "ms-vscode.makefile-tools"
+        }
+    ],
+    "version": 4
+}

Samples/EGLStreams_CUDA_Interop/.vscode/extensions.json

@@ -0,0 +1,7 @@
+{
+    "recommendations": [
+        "nvidia.nsight-vscode-edition",
+        "ms-vscode.cpptools",
+        "ms-vscode.makefile-tools"
+    ]
+}

Samples/EGLStreams_CUDA_Interop/.vscode/launch.json

@@ -0,0 +1,10 @@
+{
+    "configurations": [
+        {
+            "name": "CUDA C++: Launch",
+            "type": "cuda-gdb",
+            "request": "launch",
+            "program": "${workspaceFolder}/EGLStream_CUDA_Interop"
+        }
+    ]
+}

Samples/EGLStreams_CUDA_Interop/.vscode/tasks.json

@@ -0,0 +1,15 @@
+{
+    "version": "2.0.0",
+    "tasks": [
+        {
+            "label": "sample",
+            "type": "shell",
+            "command": "make dbg=1",
+            "problemMatcher": ["$nvcc"],
+            "group": {
+                "kind": "build",
+                "isDefault": true
+            }
+        }
+    ]
+}

Samples/EGLStreams_CUDA_Interop/Makefile

@@ -0,0 +1,433 @@
+################################################################################
+# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#  * Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#  * Neither the name of NVIDIA CORPORATION nor the names of its
+#    contributors may be used to endorse or promote products derived
+#    from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+################################################################################
+#
+# Makefile project only supported on Mac OS X and Linux Platforms)
+#
+################################################################################
+
+# Location of the CUDA Toolkit
+CUDA_PATH ?= /usr/local/cuda
+
+##############################
+# start deprecated interface #
+##############################
+ifeq ($(x86_64),1)
+    $(info WARNING - x86_64 variable has been deprecated)
+    $(info WARNING - please use TARGET_ARCH=x86_64 instead)
+    TARGET_ARCH ?= x86_64
+endif
+ifeq ($(ARMv7),1)
+    $(info WARNING - ARMv7 variable has been deprecated)
+    $(info WARNING - please use TARGET_ARCH=armv7l instead)
+    TARGET_ARCH ?= armv7l
+endif
+ifeq ($(aarch64),1)
+    $(info WARNING - aarch64 variable has been deprecated)
+    $(info WARNING - please use TARGET_ARCH=aarch64 instead)
+    TARGET_ARCH ?= aarch64
+endif
+ifeq ($(ppc64le),1)
+    $(info WARNING - ppc64le variable has been deprecated)
+    $(info WARNING - please use TARGET_ARCH=ppc64le instead)
+    TARGET_ARCH ?= ppc64le
+endif
+ifneq ($(GCC),)
+    $(info WARNING - GCC variable has been deprecated)
+    $(info WARNING - please use HOST_COMPILER=$(GCC) instead)
+    HOST_COMPILER ?= $(GCC)
+endif
+ifneq ($(abi),)
+    $(error ERROR - abi variable has been removed)
+endif
+############################
+# end deprecated interface #
+############################
+
+# architecture
+HOST_ARCH   := $(shell uname -m)
+TARGET_ARCH ?= $(HOST_ARCH)
+ifneq (,$(filter $(TARGET_ARCH),x86_64 aarch64 sbsa ppc64le armv7l))
+    ifneq ($(TARGET_ARCH),$(HOST_ARCH))
+        ifneq (,$(filter $(TARGET_ARCH),x86_64 aarch64 sbsa ppc64le))
+            TARGET_SIZE := 64
+        else ifneq (,$(filter $(TARGET_ARCH),armv7l))
+            TARGET_SIZE := 32
+        endif
+    else
+        TARGET_SIZE := $(shell getconf LONG_BIT)
+    endif
+else
+    $(error ERROR - unsupported value $(TARGET_ARCH) for TARGET_ARCH!)
+endif
+
+# sbsa and aarch64 systems look similar. Need to differentiate them at host level for now.
+ifeq ($(HOST_ARCH),aarch64)
+    ifeq ($(CUDA_PATH)/targets/sbsa-linux,$(shell ls -1d $(CUDA_PATH)/targets/sbsa-linux 2>/dev/null))
+        HOST_ARCH := sbsa
+        TARGET_ARCH := sbsa
+    endif
+endif
+
+ifneq ($(TARGET_ARCH),$(HOST_ARCH))
+    ifeq (,$(filter $(HOST_ARCH)-$(TARGET_ARCH),aarch64-armv7l x86_64-armv7l x86_64-aarch64 x86_64-sbsa x86_64-ppc64le))
+        $(error ERROR - cross compiling from $(HOST_ARCH) to $(TARGET_ARCH) is not supported!)
+    endif
+endif
+
+# When on native aarch64 system with userspace of 32-bit, change TARGET_ARCH to armv7l
+ifeq ($(HOST_ARCH)-$(TARGET_ARCH)-$(TARGET_SIZE),aarch64-aarch64-32)
+    TARGET_ARCH = armv7l
+endif
+
+# operating system
+HOST_OS   := $(shell uname -s 2>/dev/null | tr "[:upper:]" "[:lower:]")
+TARGET_OS ?= $(HOST_OS)
+ifeq (,$(filter $(TARGET_OS),linux darwin qnx android))
+    $(error ERROR - unsupported value $(TARGET_OS) for TARGET_OS!)
+endif
+
+# host compiler
+ifeq ($(TARGET_OS),darwin)
+    ifeq ($(shell expr `xcodebuild -version | grep -i xcode | awk '{print $$2}' | cut -d'.' -f1` \>= 5),1)
+        HOST_COMPILER ?= clang++
+    endif
+else ifneq ($(TARGET_ARCH),$(HOST_ARCH))
+    ifeq ($(HOST_ARCH)-$(TARGET_ARCH),x86_64-armv7l)
+        ifeq ($(TARGET_OS),linux)
+            HOST_COMPILER ?= arm-linux-gnueabihf-g++
+        else ifeq ($(TARGET_OS),qnx)
+            ifeq ($(QNX_HOST),)
+                $(error ERROR - QNX_HOST must be passed to the QNX host toolchain)
+            endif
+            ifeq ($(QNX_TARGET),)
+                $(error ERROR - QNX_TARGET must be passed to the QNX target toolchain)
+            endif
+            export QNX_HOST
+            export QNX_TARGET
+            HOST_COMPILER ?= $(QNX_HOST)/usr/bin/arm-unknown-nto-qnx6.6.0eabi-g++
+        else ifeq ($(TARGET_OS),android)
+            HOST_COMPILER ?= arm-linux-androideabi-g++
+        endif
+    else ifeq ($(TARGET_ARCH),aarch64)
+        ifeq ($(TARGET_OS), linux)
+            HOST_COMPILER ?= aarch64-linux-gnu-g++
+        else ifeq ($(TARGET_OS),qnx)
+            ifeq ($(QNX_HOST),)
+                $(error ERROR - QNX_HOST must be passed to the QNX host toolchain)
+            endif
+            ifeq ($(QNX_TARGET),)
+                $(error ERROR - QNX_TARGET must be passed to the QNX target toolchain)
+            endif
+            export QNX_HOST
+            export QNX_TARGET
+            HOST_COMPILER ?= $(QNX_HOST)/usr/bin/q++
+        else ifeq ($(TARGET_OS), android)
+            HOST_COMPILER ?= aarch64-linux-android-clang++
+        endif
+    else ifeq ($(TARGET_ARCH),sbsa)
+        HOST_COMPILER ?= aarch64-linux-gnu-g++
+    else ifeq ($(TARGET_ARCH),ppc64le)
+        HOST_COMPILER ?= powerpc64le-linux-gnu-g++
+    endif
+endif
+HOST_COMPILER ?= g++
+NVCC          := $(CUDA_PATH)/bin/nvcc -ccbin $(HOST_COMPILER)
+
+# internal flags
+NVCCFLAGS   := -m${TARGET_SIZE}
+CCFLAGS     :=
+LDFLAGS     :=
+
+# build flags
+ifeq ($(TARGET_OS),darwin)
+    LDFLAGS += -rpath $(CUDA_PATH)/lib
+    CCFLAGS += -arch $(HOST_ARCH)
+else ifeq ($(HOST_ARCH)-$(TARGET_ARCH)-$(TARGET_OS),x86_64-armv7l-linux)
+    LDFLAGS += --dynamic-linker=/lib/ld-linux-armhf.so.3
+    CCFLAGS += -mfloat-abi=hard
+else ifeq ($(TARGET_OS),android)
+    LDFLAGS += -pie
+    CCFLAGS += -fpie -fpic -fexceptions
+endif
+
+ifneq ($(TARGET_ARCH),$(HOST_ARCH))
+    ifeq ($(TARGET_ARCH)-$(TARGET_OS),armv7l-linux)
+        ifneq ($(TARGET_FS),)
+            GCCVERSIONLTEQ46 := $(shell expr `$(HOST_COMPILER) -dumpversion` \<= 4.6)
+            ifeq ($(GCCVERSIONLTEQ46),1)
+                CCFLAGS += --sysroot=$(TARGET_FS)
+            endif
+            LDFLAGS += --sysroot=$(TARGET_FS)
+            LDFLAGS += -rpath-link=$(TARGET_FS)/lib
+            LDFLAGS += -rpath-link=$(TARGET_FS)/usr/lib
+            LDFLAGS += -rpath-link=$(TARGET_FS)/usr/lib/arm-linux-gnueabihf
+        endif
+    endif
+    ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-linux)
+        ifneq ($(TARGET_FS),)
+            GCCVERSIONLTEQ46 := $(shell expr `$(HOST_COMPILER) -dumpversion` \<= 4.6)
+            ifeq ($(GCCVERSIONLTEQ46),1)
+                CCFLAGS += --sysroot=$(TARGET_FS)
+            endif
+            LDFLAGS += --sysroot=$(TARGET_FS)
+            LDFLAGS += -rpath-link=$(TARGET_FS)/lib -L$(TARGET_FS)/lib
+            LDFLAGS += -rpath-link=$(TARGET_FS)/lib/aarch64-linux-gnu -L$(TARGET_FS)/lib/aarch64-linux-gnu
+            LDFLAGS += -rpath-link=$(TARGET_FS)/usr/lib -L$(TARGET_FS)/usr/lib
+            LDFLAGS += -rpath-link=$(TARGET_FS)/usr/lib/aarch64-linux-gnu -L$(TARGET_FS)/usr/lib/aarch64-linux-gnu
+            LDFLAGS += --unresolved-symbols=ignore-in-shared-libs
+            CCFLAGS += -isystem=$(TARGET_FS)/usr/include -I$(TARGET_FS)/usr/include -I$(TARGET_FS)/usr/include/libdrm
+            CCFLAGS += -isystem=$(TARGET_FS)/usr/include/aarch64-linux-gnu -I$(TARGET_FS)/usr/include/aarch64-linux-gnu
+        endif
+    endif
+    ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-qnx)
+        NVCCFLAGS += --qpp-config 5.4.0,gcc_ntoaarch64le
+        CCFLAGS += -DWIN_INTERFACE_CUSTOM -I/usr/include/aarch64-qnx-gnu
+        LDFLAGS += -lsocket
+        LDFLAGS += -L/usr/lib/aarch64-qnx-gnu
+        CCFLAGS += "-Wl\,-rpath-link\,/usr/lib/aarch64-qnx-gnu"
+        ifdef TARGET_OVERRIDE
+            LDFLAGS += -lslog2
+        endif
+
+        ifneq ($(TARGET_FS),)
+            LDFLAGS += -L$(TARGET_FS)/usr/lib
+            CCFLAGS += "-Wl\,-rpath-link\,$(TARGET_FS)/usr/lib"
+            LDFLAGS += -L$(TARGET_FS)/usr/libnvidia
+            CCFLAGS += "-Wl\,-rpath-link\,$(TARGET_FS)/usr/libnvidia"
+            CCFLAGS += -I$(TARGET_FS)/../include
+        endif
+    endif
+endif
+
+ifdef TARGET_OVERRIDE # cuda toolkit targets override
+    NVCCFLAGS += -target-dir $(TARGET_OVERRIDE)
+endif
+
+# Install directory of different arch
+CUDA_INSTALL_TARGET_DIR :=
+ifeq ($(TARGET_ARCH)-$(TARGET_OS),armv7l-linux)
+    CUDA_INSTALL_TARGET_DIR = targets/armv7-linux-gnueabihf/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-linux)
+    CUDA_INSTALL_TARGET_DIR = targets/aarch64-linux/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),sbsa-linux)
+    CUDA_INSTALL_TARGET_DIR = targets/sbsa-linux/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),armv7l-android)
+    CUDA_INSTALL_TARGET_DIR = targets/armv7-linux-androideabi/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-android)
+    CUDA_INSTALL_TARGET_DIR = targets/aarch64-linux-androideabi/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),armv7l-qnx)
+    CUDA_INSTALL_TARGET_DIR = targets/ARMv7-linux-QNX/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-qnx)
+    CUDA_INSTALL_TARGET_DIR = targets/aarch64-qnx/
+else ifeq ($(TARGET_ARCH),ppc64le)
+    CUDA_INSTALL_TARGET_DIR = targets/ppc64le-linux/
+endif
+
+# Debug build flags
+ifeq ($(dbg),1)
+      NVCCFLAGS += -g -G
+      BUILD_TYPE := debug
+else
+      BUILD_TYPE := release
+endif
+
+ALL_CCFLAGS :=
+ALL_CCFLAGS += $(NVCCFLAGS)
+ALL_CCFLAGS += $(EXTRA_NVCCFLAGS)
+ALL_CCFLAGS += $(addprefix -Xcompiler ,$(CCFLAGS))
+ALL_CCFLAGS += $(addprefix -Xcompiler ,$(EXTRA_CCFLAGS))
+
+UBUNTU = $(shell lsb_release -i -s 2>/dev/null | grep -i ubuntu)
+
+SAMPLE_ENABLED := 1
+
+# This sample is not supported on Mac OSX
+ifeq ($(TARGET_OS),darwin)
+  $(info >>> WARNING - EGLStream_CUDA_Interop is not supported on Mac OSX - waiving sample <<<)
+  SAMPLE_ENABLED := 0
+endif
+
+# This sample is not supported on ARMv7
+ifeq ($(TARGET_ARCH),armv7l)
+  $(info >>> WARNING - EGLStream_CUDA_Interop is not supported on ARMv7 - waiving sample <<<)
+  SAMPLE_ENABLED := 0
+endif
+
+# This sample is not supported on android
+ifeq ($(TARGET_OS),android)
+  $(info >>> WARNING - EGLStream_CUDA_Interop is not supported on android - waiving sample <<<)
+  SAMPLE_ENABLED := 0
+endif
+
+ALL_LDFLAGS :=
+ALL_LDFLAGS += $(ALL_CCFLAGS)
+ALL_LDFLAGS += $(addprefix -Xlinker ,$(LDFLAGS))
+ALL_LDFLAGS += $(addprefix -Xlinker ,$(EXTRA_LDFLAGS))
+
+# Common includes and paths for CUDA
+INCLUDES  := -I../../Common
+LIBRARIES :=
+
+################################################################################
+
+# Makefile include to help find EGL Libraries
+include ./findegl.mk
+
+# EGL specific libraries
+ifneq ($(TARGET_OS),darwin)
+ LIBRARIES += -lEGL
+endif
+
+#Detect if installed version of GCC supports required C++11
+ifeq ($(TARGET_OS),linux)
+    empty :=
+    space := $(empty) $(empty)
+    GCCVERSIONSTRING := $(shell expr `$(HOST_COMPILER) -dumpversion`)
+#Create version number without "."
+    GCCVERSION := $(shell expr `echo $(GCCVERSIONSTRING)` | cut -f1 -d.)
+    GCCVERSION += $(shell expr `echo $(GCCVERSIONSTRING)` | cut -f2 -d.)
+    GCCVERSION += $(shell expr `echo $(GCCVERSIONSTRING)` | cut -f3 -d.)
+# Make sure the version number has at least 3 decimals
+    GCCVERSION += 00
+# Remove spaces from the version number
+    GCCVERSION := $(subst $(space),$(empty),$(GCCVERSION))
+#$(warning $(GCCVERSION))
+
+    IS_MIN_VERSION := $(shell expr `echo $(GCCVERSION)` \>= 51000)
+
+    ifeq ($(IS_MIN_VERSION), 1)
+        $(info >>> GCC Version is greater or equal to 5.1.0 <<<)
+    else
+        $(info >>> Waiving build. Minimum GCC version required is 5.1.0<<<)
+        SAMPLE_ENABLED := 0
+    endif
+endif
+
+ifeq ($(TARGET_OS),darwin)
+  ALL_LDFLAGS += -Xcompiler -F/Library/Frameworks -Xlinker -framework -Xlinker CUDA
+else
+  ifeq ($(TARGET_ARCH),x86_64)
+    CUDA_SEARCH_PATH ?= $(CUDA_PATH)/lib64/stubs
+    CUDA_SEARCH_PATH += $(CUDA_PATH)/targets/x86_64-linux/lib/stubs
+  endif
+
+  ifeq ($(TARGET_ARCH)-$(TARGET_OS),armv7l-linux)
+    CUDA_SEARCH_PATH ?= $(CUDA_PATH)/targets/armv7-linux-gnueabihf/lib/stubs
+  endif
+
+  ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-linux)
+    CUDA_SEARCH_PATH ?= $(CUDA_PATH)/targets/aarch64-linux/lib/stubs
+  endif
+
+  ifeq ($(TARGET_ARCH)-$(TARGET_OS),sbsa-linux)
+    CUDA_SEARCH_PATH ?= $(CUDA_PATH)/targets/sbsa-linux/lib/stubs
+  endif
+
+  ifeq ($(TARGET_ARCH)-$(TARGET_OS),armv7l-android)
+    CUDA_SEARCH_PATH ?= $(CUDA_PATH)/targets/armv7-linux-androideabi/lib/stubs
+  endif
+
+  ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-android)
+    CUDA_SEARCH_PATH ?= $(CUDA_PATH)/targets/aarch64-linux-androideabi/lib/stubs
+  endif
+
+  ifeq ($(TARGET_ARCH)-$(TARGET_OS),armv7l-qnx)
+    CUDA_SEARCH_PATH ?= $(CUDA_PATH)/targets/ARMv7-linux-QNX/lib/stubs
+  endif
+
+  ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-qnx)
+    CUDA_SEARCH_PATH ?= $(CUDA_PATH)/targets/aarch64-qnx/lib/stubs
+    ifdef TARGET_OVERRIDE
+        CUDA_SEARCH_PATH := $(CUDA_PATH)/targets/$(TARGET_OVERRIDE)/lib/stubs
+    endif
+  endif
+
+  ifeq ($(TARGET_ARCH),ppc64le)
+    CUDA_SEARCH_PATH ?= $(CUDA_PATH)/targets/ppc64le-linux/lib/stubs
+  endif
+
+  ifeq ($(HOST_ARCH),ppc64le)
+    CUDA_SEARCH_PATH += $(CUDA_PATH)/lib64/stubs
+  endif
+
+  CUDALIB ?= $(shell find -L $(CUDA_SEARCH_PATH) -maxdepth 1 -name libcuda.so 2> /dev/null)
+  ifeq ("$(CUDALIB)","")
+    $(info >>> WARNING - libcuda.so not found, CUDA Driver is not installed.  Please re-install the driver. <<<)
+    SAMPLE_ENABLED := 0
+  else
+    CUDALIB := $(shell echo $(CUDALIB) | sed "s/ .*//" | sed "s/\/libcuda.so//" )
+    LIBRARIES += -L$(CUDALIB) -lcuda
+  endif
+endif
+
+ALL_CCFLAGS += --threads 0 --std=c++11
+
+ifeq ($(SAMPLE_ENABLED),0)
+EXEC ?= @echo "[@]"
+endif
+
+################################################################################
+
+# Target rules
+all: build
+
+build: EGLStream_CUDA_Interop
+
+check.deps:
+ifeq ($(SAMPLE_ENABLED),0)
+	@echo "Sample will be waived due to the above missing dependencies"
+else
+	@echo "Sample is ready - all dependencies have been met"
+endif
+
+cuda_consumer.o:cuda_consumer.cpp
+	$(EXEC) $(NVCC) $(INCLUDES) $(ALL_CCFLAGS) $(GENCODE_FLAGS) -o $@ -c $<
+
+cuda_producer.o:cuda_producer.cpp
+	$(EXEC) $(NVCC) $(INCLUDES) $(ALL_CCFLAGS) $(GENCODE_FLAGS) -o $@ -c $<
+
+eglstrm_common.o:eglstrm_common.cpp
+	$(EXEC) $(NVCC) $(INCLUDES) $(ALL_CCFLAGS) $(GENCODE_FLAGS) -o $@ -c $<
+
+main.o:main.cpp
+	$(EXEC) $(NVCC) $(INCLUDES) $(ALL_CCFLAGS) $(GENCODE_FLAGS) -o $@ -c $<
+
+EGLStream_CUDA_Interop: cuda_consumer.o cuda_producer.o eglstrm_common.o main.o
+	$(EXEC) $(NVCC) $(ALL_LDFLAGS) $(GENCODE_FLAGS) -o $@ $+ $(LIBRARIES)
+	$(EXEC) mkdir -p ../../bin/$(TARGET_ARCH)/$(TARGET_OS)/$(BUILD_TYPE)
+	$(EXEC) cp $@ ../../bin/$(TARGET_ARCH)/$(TARGET_OS)/$(BUILD_TYPE)
+
+run: build
+	$(EXEC) ./EGLStream_CUDA_Interop
+
+clean:
+	rm -f EGLStream_CUDA_Interop cuda_consumer.o cuda_producer.o eglstrm_common.o main.o
+	rm -rf ../../bin/$(TARGET_ARCH)/$(TARGET_OS)/$(BUILD_TYPE)/EGLStream_CUDA_Interop
+
+clobber: clean

Samples/EGLStreams_CUDA_Interop/NsightEclipse.xml

@@ -0,0 +1,77 @@
+<?xml version="1.0" encoding="UTF-8"?> 
+<!DOCTYPE entry SYSTEM "SamplesInfo.dtd">
+<entry>
+  <name>EGLStream_CUDA_Interop</name>
+  <cuda_api_list>
+    <driver>cuDeviceGet</driver>
+    <driver>cuDeviceGetAttribute</driver>
+    <driver>cuDeviceComputeCapability</driver>
+    <driver>cuDeviceGetCount</driver>
+    <driver>cuDeviceGetName</driver>
+    <driver>cuGraphicsResourceGetMappedEglFrame</driver>
+    <driver>cuEGLStreamConsumerAcquireFrame</driver>
+    <driver>cuEGLStreamConsumerReleaseFrame</driver>
+    <driver>cuEGLStreamProducerPresentFrame</driver>
+    <driver>cuCtxCreate</driver>
+    <driver>cuMemAlloc</driver>
+    <driver>cuMemFree</driver>
+    <driver>cuMemcpy3D</driver>
+    <driver>cuStreamCreate</driver>
+    <driver>cuCtxPushCurrent</driver>
+    <driver>cuCtxPopCurrent</driver>
+    <driver>cuCtxDestroy</driver>
+  </cuda_api_list>
+  <description><![CDATA[Demonstrates data exchange between CUDA and EGL Streams.]]></description>
+  <devicecompilation>whole</devicecompilation>
+  <includepaths>
+    <path>./</path>
+    <path>../</path>
+    <path>../../Common</path>
+  </includepaths>
+  <keyconcepts>
+    <concept level="basic">EGLStreams Interop</concept>
+  </keyconcepts>
+  <keywords>
+    <keyword>EGL Streams</keyword>
+  </keywords>
+  <libraries>
+    <library os="linux">cuda</library>
+    <library framework="true" os="macosx">CUDA</library>
+  </libraries>
+  <librarypaths>
+  </librarypaths>
+  <nsight_eclipse>true</nsight_eclipse>
+  <primary_file>main.cpp</primary_file>
+  <required_dependencies>
+    <dependency>EGL</dependency>
+  </required_dependencies>
+  <scopes>
+    <scope>1:CUDA Basic Topics</scope>
+    <scope>2:Graphics Interop</scope>
+  </scopes>
+  <sm-arch>sm35</sm-arch>
+  <sm-arch>sm37</sm-arch>
+  <sm-arch>sm50</sm-arch>
+  <sm-arch>sm52</sm-arch>
+  <sm-arch>sm60</sm-arch>
+  <sm-arch>sm61</sm-arch>
+  <sm-arch>sm70</sm-arch>
+  <sm-arch>sm72</sm-arch>
+  <sm-arch>sm75</sm-arch>
+  <sm-arch>sm80</sm-arch>
+  <sm-arch>sm86</sm-arch>
+  <supported_envs>
+    <env>
+      <arch>x86_64</arch>
+      <platform>linux</platform>
+    </env>
+    <env>
+      <arch>aarch64</arch>
+    </env>
+  </supported_envs>
+  <supported_sm_architectures>
+    <include>all</include>
+  </supported_sm_architectures>
+  <title>EGLStream CUDA Interop</title>
+  <type>exe</type>
+</entry>

Samples/EGLStreams_CUDA_Interop/README.md

@@ -0,0 +1,64 @@
+# EGLStream_CUDA_Interop - EGLStream CUDA Interop
+
+## Description
+
+Demonstrates data exchange between CUDA and EGL Streams.
+
+## Key Concepts
+
+EGLStreams Interop
+
+## Supported SM Architectures
+
+[SM 3.5 ](https://developer.nvidia.com/cuda-gpus)  [SM 3.7 ](https://developer.nvidia.com/cuda-gpus)  [SM 5.0 ](https://developer.nvidia.com/cuda-gpus)  [SM 5.2 ](https://developer.nvidia.com/cuda-gpus)  [SM 6.0 ](https://developer.nvidia.com/cuda-gpus)  [SM 6.1 ](https://developer.nvidia.com/cuda-gpus)  [SM 7.0 ](https://developer.nvidia.com/cuda-gpus)  [SM 7.2 ](https://developer.nvidia.com/cuda-gpus)  [SM 7.5 ](https://developer.nvidia.com/cuda-gpus)  [SM 8.0 ](https://developer.nvidia.com/cuda-gpus)  [SM 8.6 ](https://developer.nvidia.com/cuda-gpus)
+
+## Supported OSes
+
+Linux
+
+## Supported CPU Architecture
+
+x86_64, aarch64
+
+## CUDA APIs involved
+
+### [CUDA Driver API](http://docs.nvidia.com/cuda/cuda-driver-api/index.html)
+cuDeviceGet, cuDeviceGetAttribute, cuDeviceComputeCapability, cuDeviceGetCount, cuDeviceGetName, cuGraphicsResourceGetMappedEglFrame, cuEGLStreamConsumerAcquireFrame, cuEGLStreamConsumerReleaseFrame, cuEGLStreamProducerPresentFrame, cuCtxCreate, cuMemAlloc, cuMemFree, cuMemcpy3D, cuStreamCreate, cuCtxPushCurrent, cuCtxPopCurrent, cuCtxDestroy
+
+## Dependencies needed to build/run
+[EGL](../../README.md#egl)
+
+## Prerequisites
+
+Download and install the [CUDA Toolkit 11.5](https://developer.nvidia.com/cuda-downloads) for your corresponding platform.
+Make sure the dependencies mentioned in [Dependencies]() section above are installed.
+
+## Build and Run
+
+### Linux
+The Linux samples are built using makefiles. To use the makefiles, change the current directory to the sample directory you wish to build, and run make:
+```
+$ cd <sample_dir>
+$ make
+```
+The samples makefiles can take advantage of certain options:
+*  **TARGET_ARCH=<arch>** - cross-compile targeting a specific architecture. Allowed architectures are x86_64, aarch64.
+    By default, TARGET_ARCH is set to HOST_ARCH. On a x86_64 machine, not setting TARGET_ARCH is the equivalent of setting TARGET_ARCH=x86_64.<br/>
+`$ make TARGET_ARCH=x86_64` <br/> `$ make TARGET_ARCH=aarch64` <br/>
+    See [here](http://docs.nvidia.com/cuda/cuda-samples/index.html#cross-samples) for more details.
+*   **dbg=1** - build with debug symbols
+    ```
+    $ make dbg=1
+    ```
+*   **SMS="A B ..."** - override the SM architectures for which the sample will be built, where `"A B ..."` is a space-delimited list of SM architectures. For example, to generate SASS for SM 50 and SM 60, use `SMS="50 60"`.
+    ```
+    $ make SMS="50 60"
+    ```
+
+*  **HOST_COMPILER=<host_compiler>** - override the default g++ host compiler. See the [Linux Installation Guide](http://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html#system-requirements) for a list of supported host compilers.
+```
+    $ make HOST_COMPILER=g++
+```
+
+## References (for more details)
+

Samples/EGLStreams_CUDA_Interop/cuda_consumer.cpp

@@ -0,0 +1,318 @@
+/* Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+//
+// DESCRIPTION:   Simple CUDA consumer rendering sample app
+//
+
+#include "cuda_consumer.h"
+#include <helper_cuda_drvapi.h>
+#include "eglstrm_common.h"
+
+#if defined(EXTENSION_LIST)
+EXTENSION_LIST(EXTLST_EXTERN)
+#endif
+
+int checkbuf(FILE *fp1, FILE *fp2);
+
+CUresult cudaConsumerTest(test_cuda_consumer_s *data, char *fileName) {
+  CUresult cuStatus = CUDA_SUCCESS;
+  CUarray cudaArr = NULL;
+  CUeglFrame cudaEgl;
+  CUgraphicsResource cudaResource;
+  unsigned int i;
+  int check_result;
+  FILE *pInFile1 = NULL, *pInFile2 = NULL, *file_p = NULL;
+  EGLint streamState = 0;
+
+  if (!data) {
+    printf("%s: Bad parameter\n", __func__);
+    goto done;
+  }
+
+  if (!eglQueryStreamKHR(g_display, eglStream, EGL_STREAM_STATE_KHR,
+                         &streamState)) {
+    printf("Cuda consumer, eglQueryStreamKHR EGL_STREAM_STATE_KHR failed\n");
+  }
+  if (streamState == EGL_STREAM_STATE_DISCONNECTED_KHR) {
+    printf("CUDA Consumer: - EGL_STREAM_STATE_DISCONNECTED_KHR received\n");
+  }
+
+  if (streamState == EGL_STREAM_STATE_NEW_FRAME_AVAILABLE_KHR) {
+    cuStatus = cuEGLStreamConsumerAcquireFrame(&(data->cudaConn), &cudaResource,
+                                               NULL, 16000);
+
+    if (cuStatus == CUDA_SUCCESS) {
+      CUdeviceptr pDevPtr = 0;
+      int bufferSize;
+      unsigned char *pCudaCopyMem = NULL;
+      unsigned int copyWidthInBytes = 0, copyHeight = 0;
+
+      file_p = fopen(fileName, "wb+");
+      if (!file_p) {
+        printf("WriteFrame: file open failed %s\n", fileName);
+        cuStatus = CUDA_ERROR_UNKNOWN;
+        goto done;
+      }
+      cuStatus =
+          cuGraphicsResourceGetMappedEglFrame(&cudaEgl, cudaResource, 0, 0);
+      if (cuStatus != CUDA_SUCCESS) {
+        printf("Cuda get resource failed with %d\n", cuStatus);
+        goto done;
+      }
+      cuStatus = cuCtxSynchronize();
+      if (cuStatus != CUDA_SUCCESS) {
+        printf("cuCtxSynchronize failed \n");
+        goto done;
+      }
+      if (!(cudaEgl.planeCount >= 1 && cudaEgl.planeCount <= 3)) {
+        printf("Plane count is invalid\nExiting\n");
+        goto done;
+      }
+
+      for (i = 0; i < cudaEgl.planeCount; i++) {
+        if (cudaEgl.frameType == CU_EGL_FRAME_TYPE_PITCH) {
+          pDevPtr = (CUdeviceptr)cudaEgl.frame.pPitch[i];
+          if (cudaEgl.planeCount == 1) {
+            bufferSize = cudaEgl.pitch * cudaEgl.height;
+            copyWidthInBytes = cudaEgl.pitch;
+            copyHeight = data->height;
+          } else if (i == 1 && cudaEgl.planeCount == 2) {  // YUV 420
+                                                           // semi-planar
+            bufferSize = cudaEgl.pitch * cudaEgl.height / 2;
+            copyWidthInBytes = cudaEgl.pitch;
+            copyHeight = data->height / 2;
+          } else {
+            bufferSize = data->width * data->height;
+            copyWidthInBytes = data->width;
+            copyHeight = data->height;
+            if (i > 0) {
+              bufferSize >>= 2;
+              copyWidthInBytes >>= 1;
+              copyHeight >>= 1;
+            }
+          }
+        } else {
+          cudaArr = cudaEgl.frame.pArray[i];
+          if (cudaEgl.planeCount == 1) {
+            bufferSize = data->width * data->height * 4;
+            copyWidthInBytes = data->width * 4;
+            copyHeight = data->height;
+          } else if (i == 1 && cudaEgl.planeCount == 2) {  // YUV 420
+                                                           // semi-planar
+            bufferSize = data->width * data->height / 2;
+            copyWidthInBytes = data->width;
+            copyHeight = data->height / 2;
+          } else {
+            bufferSize = data->width * data->height;
+            copyWidthInBytes = data->width;
+            copyHeight = data->height;
+            if (i > 0) {
+              bufferSize >>= 2;
+              copyWidthInBytes >>= 1;
+              copyHeight >>= 1;
+            }
+          }
+        }
+        if (i == 0) {
+          pCudaCopyMem = (unsigned char *)malloc(bufferSize);
+          if (pCudaCopyMem == NULL) {
+            printf("pCudaCopyMem malloc failed\n");
+            goto done;
+          }
+        }
+        memset(pCudaCopyMem, 0, bufferSize);
+        if (data->pitchLinearOutput) {
+          cuStatus = cuMemcpyDtoH(pCudaCopyMem, pDevPtr, bufferSize);
+          if (cuStatus != CUDA_SUCCESS) {
+            printf(
+                "cuda_consumer: pitch linear Memcpy failed, bufferSize =%d\n",
+                bufferSize);
+            goto done;
+          }
+          cuStatus = cuCtxSynchronize();
+          if (cuStatus != CUDA_SUCCESS) {
+            printf("cuda_consumer: cuCtxSynchronize failed after memcpy \n");
+            goto done;
+          }
+        } else {
+          CUDA_MEMCPY3D cpdesc;
+          memset(&cpdesc, 0, sizeof(cpdesc));
+          cpdesc.srcXInBytes = cpdesc.srcY = cpdesc.srcZ = cpdesc.srcLOD = 0;
+          cpdesc.srcMemoryType = CU_MEMORYTYPE_ARRAY;
+          cpdesc.srcArray = cudaArr;
+          cpdesc.dstXInBytes = cpdesc.dstY = cpdesc.dstZ = cpdesc.dstLOD = 0;
+          cpdesc.dstMemoryType = CU_MEMORYTYPE_HOST;
+          cpdesc.dstHost = (void *)pCudaCopyMem;
+          cpdesc.WidthInBytes = copyWidthInBytes;  // data->width * 4;
+          cpdesc.Height = copyHeight;              // data->height;
+          cpdesc.Depth = 1;
+
+          cuStatus = cuMemcpy3D(&cpdesc);
+          if (cuStatus != CUDA_SUCCESS) {
+            printf(
+                "Cuda consumer: cuMemCpy3D failed,  copyWidthInBytes=%d, "
+                "copyHight=%d\n",
+                copyWidthInBytes, copyHeight);
+          }
+          cuStatus = cuCtxSynchronize();
+          if (cuStatus != CUDA_SUCCESS) {
+            printf("cuCtxSynchronize failed after memcpy \n");
+          }
+        }
+        if (cuStatus == CUDA_SUCCESS) {
+          if (fwrite(pCudaCopyMem, bufferSize, 1, file_p) != 1) {
+            printf("Cuda consumer: output file write failed\n");
+            cuStatus = CUDA_ERROR_UNKNOWN;
+            goto done;
+          }
+        }
+      }
+      pInFile1 = fopen(data->fileName1, "rb");
+      if (!pInFile1) {
+        printf("Failed to open file :%s\n", data->fileName1);
+        goto done;
+      }
+      pInFile2 = fopen(data->fileName2, "rb");
+      if (!pInFile2) {
+        printf("Failed to open file :%s\n", data->fileName2);
+        goto done;
+      }
+      rewind(file_p);
+      check_result = checkbuf(file_p, pInFile1);
+      if (check_result == -1) {
+        rewind(file_p);
+        check_result = checkbuf(file_p, pInFile2);
+        if (check_result == -1) {
+          printf("Frame received does not match any valid image: FAILED\n");
+        } else {
+          printf("Frame check Passed\n");
+        }
+      } else {
+        printf("Frame check Passed\n");
+      }
+      if (pCudaCopyMem) {
+        free(pCudaCopyMem);
+        pCudaCopyMem = NULL;
+      }
+      cuStatus =
+          cuEGLStreamConsumerReleaseFrame(&data->cudaConn, cudaResource, NULL);
+      if (cuStatus != CUDA_SUCCESS) {
+        printf("cuEGLStreamConsumerReleaseFrame failed with cuStatus = %d\n",
+               cuStatus);
+        goto done;
+      }
+    } else {
+      printf("cuda AcquireFrame FAILED with  cuStatus=%d\n", cuStatus);
+      goto done;
+    }
+  }
+
+done:
+  if (file_p) {
+    fclose(file_p);
+    file_p = NULL;
+  }
+  if (pInFile1) {
+    fclose(pInFile1);
+    pInFile1 = NULL;
+  }
+  if (pInFile1) {
+    fclose(pInFile2);
+    pInFile2 = NULL;
+  }
+  return cuStatus;
+}
+
+int checkbuf(FILE *fp1, FILE *fp2) {
+  int match = 0;
+  int ch1, ch2;
+  if (fp1 == NULL) {
+    printf("Invalid file pointer for first file\n");
+    return -1;
+  } else if (fp2 == NULL) {
+    printf("Invalid file pointer for second file\n");
+    return -1;
+  } else {
+    ch1 = getc(fp1);
+    ch2 = getc(fp2);
+    while ((ch1 != EOF) && (ch2 != EOF) && (ch1 == ch2)) {
+      ch1 = getc(fp1);
+      ch2 = getc(fp2);
+    }
+    if (ch1 == ch2) {
+      match = 1;
+    } else if (ch1 != ch2) {
+      match = -1;
+    }
+  }
+  return match;
+}
+
+CUresult cudaDeviceCreateConsumer(test_cuda_consumer_s *cudaConsumer,
+                                  CUdevice device) {
+  CUresult status = CUDA_SUCCESS;
+  if (CUDA_SUCCESS != (status = cuInit(0))) {
+    printf("Failed to initialize CUDA\n");
+    return status;
+  }
+
+  int major = 0, minor = 0;
+  char deviceName[256];
+  checkCudaErrors(cuDeviceGetAttribute(
+      &major, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR, device));
+  checkCudaErrors(cuDeviceGetAttribute(
+      &minor, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR, device));
+  checkCudaErrors(cuDeviceGetName(deviceName, 256, device));
+  printf(
+      "CUDA Consumer on GPU Device %d: \"%s\" with compute capability "
+      "%d.%d\n\n",
+      device, deviceName, major, minor);
+
+  if (CUDA_SUCCESS !=
+      (status = cuCtxCreate(&cudaConsumer->context, 0, device))) {
+    printf("failed to create CUDA context\n");
+    return status;
+  }
+  checkCudaErrors(cuCtxPopCurrent(&cudaConsumer->context));
+  return status;
+}
+
+void cuda_consumer_init(test_cuda_consumer_s *cudaConsumer, TestArgs *args) {
+  cudaConsumer->pitchLinearOutput = args->pitchLinearOutput;
+  cudaConsumer->width = args->inputWidth;
+  cudaConsumer->height = args->inputHeight;
+  cudaConsumer->fileName1 = args->infile1;
+  cudaConsumer->fileName2 = args->infile2;
+
+  cudaConsumer->outFile1 = "cuda_out1.yuv";
+  cudaConsumer->outFile2 = "cuda_out2.yuv";
+}
+
+CUresult cuda_consumer_deinit(test_cuda_consumer_s *cudaConsumer) {
+  return cuEGLStreamConsumerDisconnect(&cudaConsumer->cudaConn);
+}

Samples/EGLStreams_CUDA_Interop/cuda_consumer.h

@@ -0,0 +1,62 @@
+/* Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+//
+// DESCRIPTION:   CUDA consumer header file
+//
+
+#ifndef _CUDA_CONSUMER_H_
+#define _CUDA_CONSUMER_H_
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include "cudaEGL.h"
+#include "eglstrm_common.h"
+
+extern EGLStreamKHR eglStream;
+extern EGLDisplay g_display;
+
+typedef struct _test_cuda_consumer_s {
+  CUcontext context;
+  CUeglStreamConnection cudaConn;
+  bool pitchLinearOutput;
+  unsigned int width;
+  unsigned int height;
+  char *fileName1;
+  char *fileName2;
+  char *outFile1;
+  char *outFile2;
+  unsigned int frameCount;
+} test_cuda_consumer_s;
+
+void cuda_consumer_init(test_cuda_consumer_s *cudaConsumer, TestArgs *args);
+CUresult cuda_consumer_deinit(test_cuda_consumer_s *cudaConsumer);
+CUresult cudaConsumerTest(test_cuda_consumer_s *data, char *outFile);
+CUresult cudaDeviceCreateConsumer(test_cuda_consumer_s *cudaConsumer,
+                                  CUdevice device);
+#endif

Samples/EGLStreams_CUDA_Interop/cuda_producer.cpp

@@ -0,0 +1,406 @@
+/* Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+//
+// DESCRIPTION:   Simple cuda EGL stream producer app
+//
+
+#include "cuda_producer.h"
+#include <helper_cuda_drvapi.h>
+#include "cudaEGL.h"
+#include "eglstrm_common.h"
+
+#if defined(EXTENSION_LIST)
+EXTENSION_LIST(EXTLST_EXTERN)
+#endif
+
+static CUresult cudaProducerReadYUVFrame(FILE *file, unsigned int frameNum,
+                                         unsigned int width,
+                                         unsigned int height,
+                                         unsigned char *pBuff) {
+  int bOrderUV = 0;
+  unsigned char *pYBuff, *pUBuff, *pVBuff, *pChroma;
+  unsigned int frameSize = (width * height * 3) / 2;
+  CUresult ret = CUDA_SUCCESS;
+  unsigned int i;
+
+  if (!pBuff || !file) return CUDA_ERROR_FILE_NOT_FOUND;
+
+  pYBuff = pBuff;
+
+  // YVU order in the buffer
+  pVBuff = pYBuff + width * height;
+  pUBuff = pVBuff + width * height / 4;
+
+  if (fseek(file, frameNum * frameSize, SEEK_SET)) {
+    printf("ReadYUVFrame: Error seeking file: %p\n", file);
+    ret = CUDA_ERROR_NOT_PERMITTED;
+    goto done;
+  }
+  // read Y U V separately
+  for (i = 0; i < height; i++) {
+    if (fread(pYBuff, width, 1, file) != 1) {
+      printf("ReadYUVFrame: Error reading file: %p\n", file);
+      ret = CUDA_ERROR_NOT_PERMITTED;
+      goto done;
+    }
+    pYBuff += width;
+  }
+
+  pChroma = bOrderUV ? pUBuff : pVBuff;
+  for (i = 0; i < height / 2; i++) {
+    if (fread(pChroma, width / 2, 1, file) != 1) {
+      printf("ReadYUVFrame: Error reading file: %p\n", file);
+      ret = CUDA_ERROR_NOT_PERMITTED;
+      goto done;
+    }
+    pChroma += width / 2;
+  }
+
+  pChroma = bOrderUV ? pVBuff : pUBuff;
+  for (i = 0; i < height / 2; i++) {
+    if (fread(pChroma, width / 2, 1, file) != 1) {
+      printf("ReadYUVFrame: Error reading file: %p\n", file);
+      ret = CUDA_ERROR_NOT_PERMITTED;
+      goto done;
+    }
+    pChroma += width / 2;
+  }
+done:
+  return ret;
+}
+
+static CUresult cudaProducerReadARGBFrame(FILE *file, unsigned int frameNum,
+                                          unsigned int width,
+                                          unsigned int height,
+                                          unsigned char *pBuff) {
+  unsigned int frameSize = width * height * 4;
+  CUresult ret = CUDA_SUCCESS;
+
+  if (!pBuff || !file) return CUDA_ERROR_FILE_NOT_FOUND;
+
+  if (fseek(file, frameNum * frameSize, SEEK_SET)) {
+    printf("ReadYUVFrame: Error seeking file: %p\n", file);
+    ret = CUDA_ERROR_NOT_PERMITTED;
+    goto done;
+  }
+
+  // read ARGB data
+  if (fread(pBuff, frameSize, 1, file) != 1) {
+    if (feof(file))
+      printf("ReadARGBFrame: file read to the end\n");
+    else
+      printf("ReadARGBFrame: Error reading file: %p\n", file);
+    ret = CUDA_ERROR_NOT_PERMITTED;
+    goto done;
+  }
+done:
+  return ret;
+}
+
+CUresult cudaProducerTest(test_cuda_producer_s *cudaProducer, char *file) {
+  int framenum = 0;
+  CUarray cudaArr[3] = {0};
+  CUdeviceptr cudaPtr[3] = {0, 0, 0};
+  unsigned int bufferSize;
+  CUresult cuStatus = CUDA_SUCCESS;
+  unsigned int i, surfNum, uvOffset[3] = {0};
+  unsigned int copyWidthInBytes[3] = {0, 0, 0}, copyHeight[3] = {0, 0, 0};
+  CUeglColorFormat eglColorFormat;
+  FILE *file_p;
+  CUeglFrame cudaEgl;
+  CUcontext oldContext;
+
+  file_p = fopen(file, "rb");
+  if (!file_p) {
+    printf("CudaProducer: Error opening file: %s\n", file);
+    goto done;
+  }
+
+  if (cudaProducer->pitchLinearOutput) {
+    if (cudaProducer->isARGB) {
+      cudaPtr[0] = cudaProducer->cudaPtrARGB[0];
+    } else {  // YUV case
+      for (i = 0; i < 3; i++) {
+        if (i == 0) {
+          bufferSize = cudaProducer->width * cudaProducer->height;
+        } else {
+          bufferSize = cudaProducer->width * cudaProducer->height / 4;
+        }
+
+        cudaPtr[i] = cudaProducer->cudaPtrYUV[i];
+      }
+    }
+  } else {
+    if (cudaProducer->isARGB) {
+      cudaArr[0] = cudaProducer->cudaArrARGB[0];
+    } else {
+      for (i = 0; i < 3; i++) {
+        cudaArr[i] = cudaProducer->cudaArrYUV[i];
+      }
+    }
+  }
+  uvOffset[0] = 0;
+  if (cudaProducer->isARGB) {
+    if (CUDA_SUCCESS !=
+        cudaProducerReadARGBFrame(file_p, framenum, cudaProducer->width,
+                                  cudaProducer->height, cudaProducer->pBuff)) {
+      printf("cuda producer, read ARGB frame failed\n");
+      goto done;
+    }
+    copyWidthInBytes[0] = cudaProducer->width * 4;
+    copyHeight[0] = cudaProducer->height;
+    surfNum = 1;
+    eglColorFormat = CU_EGL_COLOR_FORMAT_ARGB;
+  } else {
+    if (CUDA_SUCCESS !=
+        cudaProducerReadYUVFrame(file_p, framenum, cudaProducer->width,
+                                 cudaProducer->height, cudaProducer->pBuff)) {
+      printf("cuda producer, reading YUV frame failed\n");
+      goto done;
+    }
+    surfNum = 3;
+    eglColorFormat = CU_EGL_COLOR_FORMAT_YUV420_PLANAR;
+    copyWidthInBytes[0] = cudaProducer->width;
+    copyHeight[0] = cudaProducer->height;
+    copyWidthInBytes[1] = cudaProducer->width / 2;
+    copyHeight[1] = cudaProducer->height / 2;
+    copyWidthInBytes[2] = cudaProducer->width / 2;
+    copyHeight[2] = cudaProducer->height / 2;
+    uvOffset[1] = cudaProducer->width * cudaProducer->height;
+    uvOffset[2] =
+        uvOffset[1] + cudaProducer->width / 2 * cudaProducer->height / 2;
+  }
+  if (cudaProducer->pitchLinearOutput) {
+    for (i = 0; i < surfNum; i++) {
+      cuStatus =
+          cuMemcpy(cudaPtr[i], (CUdeviceptr)(cudaProducer->pBuff + uvOffset[i]),
+                   copyWidthInBytes[i] * copyHeight[i]);
+
+      if (cuStatus != CUDA_SUCCESS) {
+        printf("Cuda producer: cuMemCpy pitchlinear failed, cuStatus =%d\n",
+               cuStatus);
+        goto done;
+      }
+    }
+  } else {
+    // copy cudaProducer->pBuff to cudaArray
+    CUDA_MEMCPY3D cpdesc;
+    for (i = 0; i < surfNum; i++) {
+      memset(&cpdesc, 0, sizeof(cpdesc));
+      cpdesc.srcXInBytes = cpdesc.srcY = cpdesc.srcZ = cpdesc.srcLOD = 0;
+      cpdesc.srcMemoryType = CU_MEMORYTYPE_HOST;
+      cpdesc.srcHost = (void *)(cudaProducer->pBuff + uvOffset[i]);
+      cpdesc.dstXInBytes = cpdesc.dstY = cpdesc.dstZ = cpdesc.dstLOD = 0;
+      cpdesc.dstMemoryType = CU_MEMORYTYPE_ARRAY;
+      cpdesc.dstArray = cudaArr[i];
+      cpdesc.WidthInBytes = copyWidthInBytes[i];
+      cpdesc.Height = copyHeight[i];
+      cpdesc.Depth = 1;
+      cuStatus = cuMemcpy3D(&cpdesc);
+      if (cuStatus != CUDA_SUCCESS) {
+        printf("Cuda producer: cuMemCpy failed, cuStatus =%d\n", cuStatus);
+        goto done;
+      }
+    }
+  }
+  for (i = 0; i < surfNum; i++) {
+    if (cudaProducer->pitchLinearOutput)
+      cudaEgl.frame.pPitch[i] = (void *)cudaPtr[i];
+    else
+      cudaEgl.frame.pArray[i] = cudaArr[i];
+  }
+  cudaEgl.width = copyWidthInBytes[0];
+  cudaEgl.depth = 1;
+  cudaEgl.height = copyHeight[0];
+  cudaEgl.pitch = cudaProducer->pitchLinearOutput ? cudaEgl.width : 0;
+  cudaEgl.frameType = cudaProducer->pitchLinearOutput ? CU_EGL_FRAME_TYPE_PITCH
+                                                      : CU_EGL_FRAME_TYPE_ARRAY;
+  cudaEgl.planeCount = surfNum;
+  cudaEgl.numChannels = (eglColorFormat == CU_EGL_COLOR_FORMAT_ARGB) ? 4 : 1;
+  cudaEgl.eglColorFormat = eglColorFormat;
+  cudaEgl.cuFormat = CU_AD_FORMAT_UNSIGNED_INT8;
+
+  static int numFramesPresented = 0;
+  // If there is a frame presented before we check if consumer
+  // is done with it using cuEGLStreamProducerReturnFrame.
+  while (numFramesPresented) {
+    CUeglFrame returnedCudaEgl;
+    cuStatus = cuEGLStreamProducerReturnFrame(&cudaProducer->cudaConn,
+                                              &returnedCudaEgl, NULL);
+    if (cuStatus == CUDA_ERROR_LAUNCH_TIMEOUT) {
+      continue;
+    } else if (cuStatus != CUDA_SUCCESS) {
+      printf("cuda Producer return frame FAILED with custatus= %d\n", cuStatus);
+      return cuStatus;
+    } else {
+      numFramesPresented--;
+    }
+  }
+
+  cuStatus =
+      cuEGLStreamProducerPresentFrame(&cudaProducer->cudaConn, cudaEgl, NULL);
+  if (cuStatus != CUDA_SUCCESS) {
+    printf("cuda Producer present frame FAILED with custatus= %d\n", cuStatus);
+    goto done;
+  }
+  numFramesPresented++;
+
+done:
+  if (file_p) {
+    fclose(file_p);
+    file_p = NULL;
+  }
+
+  return cuStatus;
+}
+
+CUresult cudaDeviceCreateProducer(test_cuda_producer_s *cudaProducer,
+                                  CUdevice device) {
+  CUresult status = CUDA_SUCCESS;
+  if (CUDA_SUCCESS != (status = cuInit(0))) {
+    printf("Failed to initialize CUDA\n");
+    return status;
+  }
+
+  int major = 0, minor = 0;
+  char deviceName[256];
+  checkCudaErrors(cuDeviceGetAttribute(
+      &major, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR, device));
+  checkCudaErrors(cuDeviceGetAttribute(
+      &minor, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR, device));
+  checkCudaErrors(cuDeviceGetName(deviceName, 256, device));
+  printf(
+      "CUDA Producer on GPU Device %d: \"%s\" with compute capability "
+      "%d.%d\n\n",
+      device, deviceName, major, minor);
+
+  if (major < 6) {
+    printf(
+        "EGLStreams_CUDA_Interop requires SM 6.0 or higher arch GPU.  "
+        "Exiting...\n");
+    exit(2);  // EXIT_WAIVED
+  }
+
+  if (CUDA_SUCCESS !=
+      (status = cuCtxCreate(&cudaProducer->context, 0, device))) {
+    printf("failed to create CUDA context\n");
+    return status;
+  }
+
+  status = cuMemAlloc(&cudaProducer->cudaPtrARGB[0], (WIDTH * HEIGHT * 4));
+  if (status != CUDA_SUCCESS) {
+    printf("Create CUDA pointer failed, cuStatus=%d\n", status);
+    return status;
+  }
+
+  status = cuMemAlloc(&cudaProducer->cudaPtrYUV[0], (WIDTH * HEIGHT));
+  if (status != CUDA_SUCCESS) {
+    printf("Create CUDA pointer failed, cuStatus=%d\n", status);
+    return status;
+  }
+  status = cuMemAlloc(&cudaProducer->cudaPtrYUV[1], (WIDTH * HEIGHT) / 4);
+  if (status != CUDA_SUCCESS) {
+    printf("Create CUDA pointer failed, cuStatus=%d\n", status);
+    return status;
+  }
+  status = cuMemAlloc(&cudaProducer->cudaPtrYUV[2], (WIDTH * HEIGHT) / 4);
+  if (status != CUDA_SUCCESS) {
+    printf("Create CUDA pointer failed, cuStatus=%d\n", status);
+    return status;
+  }
+
+  CUDA_ARRAY3D_DESCRIPTOR desc = {0};
+
+  desc.Format = CU_AD_FORMAT_UNSIGNED_INT8;
+  desc.Depth = 1;
+  desc.Flags = CUDA_ARRAY3D_SURFACE_LDST;
+  desc.NumChannels = 4;
+  desc.Width = WIDTH * 4;
+  desc.Height = HEIGHT;
+  status = cuArray3DCreate(&cudaProducer->cudaArrARGB[0], &desc);
+  if (status != CUDA_SUCCESS) {
+    printf("Create CUDA array failed, cuStatus=%d\n", status);
+    return status;
+  }
+
+  for (int i = 0; i < 3; i++) {
+    if (i == 0) {
+      desc.NumChannels = 1;
+      desc.Width = WIDTH;
+      desc.Height = HEIGHT;
+    } else {  // U/V surface as planar
+      desc.NumChannels = 1;
+      desc.Width = WIDTH / 2;
+      desc.Height = HEIGHT / 2;
+    }
+    status = cuArray3DCreate(&cudaProducer->cudaArrYUV[i], &desc);
+    if (status != CUDA_SUCCESS) {
+      printf("Create CUDA array failed, cuStatus=%d\n", status);
+      return status;
+    }
+  }
+
+  cudaProducer->pBuff = (unsigned char *)malloc((WIDTH * HEIGHT * 4));
+  if (!cudaProducer->pBuff) {
+    printf("CudaProducer: Failed to allocate image buffer\n");
+  }
+
+  checkCudaErrors(cuCtxPopCurrent(&cudaProducer->context));
+  return status;
+}
+
+void cudaProducerInit(test_cuda_producer_s *cudaProducer, EGLDisplay eglDisplay,
+                      EGLStreamKHR eglStream, TestArgs *args) {
+  cudaProducer->fileName1 = args->infile1;
+  cudaProducer->fileName2 = args->infile2;
+
+  cudaProducer->frameCount = 2;
+  cudaProducer->width = args->inputWidth;
+  cudaProducer->height = args->inputHeight;
+  cudaProducer->isARGB = args->isARGB;
+  cudaProducer->pitchLinearOutput = args->pitchLinearOutput;
+
+  // Set cudaProducer default parameters
+  cudaProducer->eglDisplay = eglDisplay;
+  cudaProducer->eglStream = eglStream;
+}
+
+CUresult cudaProducerDeinit(test_cuda_producer_s *cudaProducer) {
+  if (cudaProducer->pBuff) free(cudaProducer->pBuff);
+
+  checkCudaErrors(cuMemFree(cudaProducer->cudaPtrARGB[0]));
+  checkCudaErrors(cuMemFree(cudaProducer->cudaPtrYUV[0]));
+  checkCudaErrors(cuMemFree(cudaProducer->cudaPtrYUV[1]));
+  checkCudaErrors(cuMemFree(cudaProducer->cudaPtrYUV[2]));
+  checkCudaErrors(cuArrayDestroy(cudaProducer->cudaArrARGB[0]));
+  checkCudaErrors(cuArrayDestroy(cudaProducer->cudaArrYUV[0]));
+  checkCudaErrors(cuArrayDestroy(cudaProducer->cudaArrYUV[1]));
+  checkCudaErrors(cuArrayDestroy(cudaProducer->cudaArrYUV[2]));
+
+  return cuEGLStreamProducerDisconnect(&cudaProducer->cudaConn);
+}

Samples/EGLStreams_CUDA_Interop/cuda_producer.h

@@ -0,0 +1,68 @@
+/* Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+//
+// DESCRIPTION:   Simple cuda producer header file
+//
+
+#ifndef _CUDA_PRODUCER_H_
+#define _CUDA_PRODUCER_H_
+#include <EGL/egl.h>
+#include <EGL/eglext.h>
+#include "cudaEGL.h"
+#include "eglstrm_common.h"
+
+extern EGLStreamKHR eglStream;
+extern EGLDisplay g_display;
+
+typedef struct _test_cuda_producer_s {
+  //  Stream params
+  char *fileName1;
+  char *fileName2;
+  unsigned char *pBuff;
+  int frameCount;
+  bool isARGB;
+  bool pitchLinearOutput;
+  unsigned int width;
+  unsigned int height;
+  CUcontext context;
+  CUeglStreamConnection cudaConn;
+  CUdeviceptr cudaPtrARGB[1];
+  CUdeviceptr cudaPtrYUV[3];
+  CUarray cudaArrARGB[1];
+  CUarray cudaArrYUV[3];
+  EGLStreamKHR eglStream;
+  EGLDisplay eglDisplay;
+} test_cuda_producer_s;
+
+void cudaProducerInit(test_cuda_producer_s *cudaProducer, EGLDisplay eglDisplay,
+                      EGLStreamKHR eglStream, TestArgs *args);
+CUresult cudaProducerTest(test_cuda_producer_s *parserArg, char *file);
+CUresult cudaProducerDeinit(test_cuda_producer_s *cudaProducer);
+CUresult cudaDeviceCreateProducer(test_cuda_producer_s *cudaProducer,
+                                  CUdevice device);
+#endif

Samples/EGLStreams_CUDA_Interop/eglstrm_common.cpp

@@ -0,0 +1,139 @@
+/* Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+//
+// DESCRIPTION:   Common egl stream functions
+//
+
+#include "eglstrm_common.h"
+
+EGLStreamKHR eglStream;
+EGLDisplay g_display;
+EGLAttrib cudaIndex;
+
+#if defined(EXTENSION_LIST)
+EXTENSION_LIST(EXTLST_DECL)
+typedef void (*extlst_fnptr_t)(void);
+static struct {
+  extlst_fnptr_t *fnptr;
+  char const *name;
+} extensionList[] = {EXTENSION_LIST(EXTLST_ENTRY)};
+
+int eglSetupExtensions(void) {
+  unsigned int i;
+
+  for (i = 0; i < (sizeof(extensionList) / sizeof(*extensionList)); i++) {
+    *extensionList[i].fnptr = eglGetProcAddress(extensionList[i].name);
+    if (*extensionList[i].fnptr == NULL) {
+      printf("Couldn't get address of %s()\n", extensionList[i].name);
+      return 0;
+    }
+  }
+
+  return 1;
+}
+
+int EGLStreamInit(int *cuda_device) {
+  static const EGLint streamAttrMailboxMode[] = {EGL_SUPPORT_REUSE_NV,
+                                                 EGL_FALSE, EGL_NONE};
+  EGLBoolean eglStatus;
+#define MAX_EGL_DEVICES 4
+  EGLint numDevices = 0;
+  EGLDeviceEXT devices[MAX_EGL_DEVICES];
+  eglStatus = eglQueryDevicesEXT(MAX_EGL_DEVICES, devices, &numDevices);
+  if (eglStatus != EGL_TRUE) {
+    printf("Error querying EGL devices\n");
+    exit(EXIT_FAILURE);
+  }
+
+  if (numDevices == 0) {
+    printf("No EGL devices found.. Waiving\n");
+    eglStatus = EGL_FALSE;
+    exit(EXIT_WAIVED);
+  }
+
+  int egl_device_id = 0;
+  for (egl_device_id = 0; egl_device_id < numDevices; egl_device_id++) {
+    eglStatus = eglQueryDeviceAttribEXT(devices[egl_device_id],
+                                        EGL_CUDA_DEVICE_NV, &cudaIndex);
+    if (eglStatus == EGL_TRUE) {
+      *cuda_device = cudaIndex;  // We select first EGL-CUDA Capable device.
+      printf("Found EGL-CUDA Capable device with CUDA Device id = %d\n",
+             (int)cudaIndex);
+      break;
+    }
+  }
+
+  if (egl_device_id >= numDevices) {
+    printf("No CUDA Capable EGL Device found.. Waiving execution\n");
+    exit(EXIT_WAIVED);
+  }
+
+  g_display = eglGetPlatformDisplayEXT(EGL_PLATFORM_DEVICE_EXT,
+                                       (void *)devices[egl_device_id], NULL);
+  if (g_display == EGL_NO_DISPLAY) {
+    printf("Could not get EGL display from device. \n");
+    eglStatus = EGL_FALSE;
+    exit(EXIT_FAILURE);
+  }
+
+  eglStatus = eglInitialize(g_display, 0, 0);
+  if (!eglStatus) {
+    printf("EGL failed to initialize. \n");
+    eglStatus = EGL_FALSE;
+    exit(EXIT_FAILURE);
+  }
+
+  eglStream = eglCreateStreamKHR(g_display, streamAttrMailboxMode);
+  if (eglStream == EGL_NO_STREAM_KHR) {
+    printf("Could not create EGL stream.\n");
+    eglStatus = EGL_FALSE;
+    exit(EXIT_FAILURE);
+  }
+
+  printf("Created EGLStream %p\n", eglStream);
+
+  // Set stream attribute
+  if (!eglStreamAttribKHR(g_display, eglStream, EGL_CONSUMER_LATENCY_USEC_KHR,
+                          16000)) {
+    printf(
+        "Consumer: eglStreamAttribKHR EGL_CONSUMER_LATENCY_USEC_KHR failed\n");
+    return 0;
+  }
+  if (!eglStreamAttribKHR(g_display, eglStream,
+                          EGL_CONSUMER_ACQUIRE_TIMEOUT_USEC_KHR, 16000)) {
+    printf(
+        "Consumer: eglStreamAttribKHR EGL_CONSUMER_ACQUIRE_TIMEOUT_USEC_KHR "
+        "failed\n");
+    return 0;
+  }
+  printf("EGLStream initialized\n");
+  return 1;
+}
+
+void EGLStreamFini(void) { eglDestroyStreamKHR(g_display, eglStream); }
+#endif

Samples/EGLStreams_CUDA_Interop/eglstrm_common.h

@@ -0,0 +1,103 @@
+/* Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+//
+// DESCRIPTION:   Common EGL stream functions header file
+//
+
+#ifndef _EGLSTRM_COMMON_H_
+#define _EGLSTRM_COMMON_H_
+
+#include <signal.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/stat.h>
+#include <sys/time.h>
+#include <unistd.h>
+
+#include "cuda.h"
+#include "cudaEGL.h"
+#include "helper_cuda_drvapi.h"
+
+#define EXTENSION_LIST(T)                                                \
+  T(PFNEGLCREATESTREAMKHRPROC, eglCreateStreamKHR)                       \
+  T(PFNEGLDESTROYSTREAMKHRPROC, eglDestroyStreamKHR)                     \
+  T(PFNEGLQUERYSTREAMKHRPROC, eglQueryStreamKHR)                         \
+  T(PFNEGLQUERYSTREAMU64KHRPROC, eglQueryStreamu64KHR)                   \
+  T(PFNEGLQUERYSTREAMTIMEKHRPROC, eglQueryStreamTimeKHR)                 \
+  T(PFNEGLSTREAMATTRIBKHRPROC, eglStreamAttribKHR)                       \
+  T(PFNEGLSTREAMCONSUMERACQUIREKHRPROC, eglStreamConsumerAcquireKHR)     \
+  T(PFNEGLSTREAMCONSUMERRELEASEKHRPROC, eglStreamConsumerReleaseKHR)     \
+  T(PFNEGLSTREAMCONSUMERGLTEXTUREEXTERNALKHRPROC,                        \
+    eglStreamConsumerGLTextureExternalKHR)                               \
+  T(PFNEGLGETSTREAMFILEDESCRIPTORKHRPROC, eglGetStreamFileDescriptorKHR) \
+  T(PFNEGLQUERYDEVICESEXTPROC, eglQueryDevicesEXT)                       \
+  T(PFNEGLGETPLATFORMDISPLAYEXTPROC, eglGetPlatformDisplayEXT)           \
+  T(PFNEGLQUERYDEVICEATTRIBEXTPROC, eglQueryDeviceAttribEXT)             \
+  T(PFNEGLCREATESTREAMFROMFILEDESCRIPTORKHRPROC,                         \
+    eglCreateStreamFromFileDescriptorKHR)
+
+#define eglCreateStreamKHR my_eglCreateStreamKHR
+#define eglDestroyStreamKHR my_eglDestroyStreamKHR
+#define eglQueryStreamKHR my_eglQueryStreamKHR
+#define eglQueryStreamu64KHR my_eglQueryStreamu64KHR
+#define eglQueryStreamTimeKHR my_eglQueryStreamTimeKHR
+#define eglStreamAttribKHR my_eglStreamAttribKHR
+#define eglStreamConsumerAcquireKHR my_eglStreamConsumerAcquireKHR
+#define eglStreamConsumerReleaseKHR my_eglStreamConsumerReleaseKHR
+#define eglStreamConsumerGLTextureExternalKHR \
+  my_eglStreamConsumerGLTextureExternalKHR
+#define eglGetStreamFileDescriptorKHR my_eglGetStreamFileDescriptorKHR
+#define eglCreateStreamFromFileDescriptorKHR \
+  my_eglCreateStreamFromFileDescriptorKHR
+#define eglQueryDevicesEXT my_eglQueryDevicesEXT
+#define eglGetPlatformDisplayEXT my_eglGetPlatformDisplayEXT
+#define eglQueryDeviceAttribEXT my_eglQueryDeviceAttribEXT
+
+#define EXTLST_DECL(tx, x) tx my_##x = NULL;
+#define EXTLST_EXTERN(tx, x) extern tx my_##x;
+#define EXTLST_ENTRY(tx, x) {(extlst_fnptr_t *)&my_##x, #x},
+
+#define MAX_STRING_SIZE 256
+#define WIDTH 720
+#define HEIGHT 480
+
+typedef struct _TestArgs {
+  char *infile1;
+  char *infile2;
+  bool isARGB;
+  unsigned int inputWidth;
+  unsigned int inputHeight;
+  bool pitchLinearOutput;
+} TestArgs;
+
+int eglSetupExtensions(void);
+int EGLStreamInit(int *dev);
+void EGLStreamFini(void);
+#endif

Samples/EGLStreams_CUDA_Interop/findegl.mk

@@ -0,0 +1,160 @@
+################################################################################
+# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#  * Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#  * Neither the name of NVIDIA CORPORATION nor the names of its
+#    contributors may be used to endorse or promote products derived
+#    from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+################################################################################
+#
+#  findegl.mk is used to find the necessary EGL Libraries for specific distributions
+#               this is supported on Linux
+#
+################################################################################
+
+# Determine OS platform and unix distribution
+ifeq ("$(TARGET_OS)","linux")
+   # first search lsb_release
+   DISTRO  = $(shell lsb_release -i -s 2>/dev/null | tr "[:upper:]" "[:lower:]")
+   ifeq ("$(DISTRO)","")
+     # second search and parse /etc/issue
+     DISTRO = $(shell more /etc/issue | awk '{print $$1}' | sed '1!d' | sed -e "/^$$/d" 2>/dev/null | tr "[:upper:]" "[:lower:]")
+     # ensure data from /etc/issue is valid
+     ifeq (,$(filter $(DISTRO),ubuntu fedora red rhel centos suse))
+       DISTRO = 
+     endif
+     ifeq ("$(DISTRO)","")
+       # third, we can search in /etc/os-release or /etc/{distro}-release
+       DISTRO = $(shell awk '/ID/' /etc/*-release | sed 's/ID=//' | grep -v "VERSION" | grep -v "ID" | grep -v "DISTRIB")
+     endif
+   endif
+endif
+
+ifeq ("$(TARGET_OS)","linux")
+    # $(info) >> findegl.mk -> LINUX path <<<)
+    # Each set of Linux Distros have different paths for where to find their OpenGL libraries reside
+    UBUNTU = $(shell echo $(DISTRO) | grep -i ubuntu      >/dev/null 2>&1; echo $$?)
+    FEDORA = $(shell echo $(DISTRO) | grep -i fedora      >/dev/null 2>&1; echo $$?)
+    RHEL   = $(shell echo $(DISTRO) | grep -i 'red\|rhel' >/dev/null 2>&1; echo $$?)
+    CENTOS = $(shell echo $(DISTRO) | grep -i centos      >/dev/null 2>&1; echo $$?)
+    SUSE   = $(shell echo $(DISTRO) | grep -i 'suse\|sles' >/dev/null 2>&1; echo $$?)
+    ifeq ("$(UBUNTU)","0")
+      ifeq ($(HOST_ARCH)-$(TARGET_ARCH),x86_64-armv7l)
+        GLPATH := /usr/arm-linux-gnueabihf/lib
+        GLLINK := -L/usr/arm-linux-gnueabihf/lib
+        ifneq ($(TARGET_FS),) 
+          GLPATH += $(TARGET_FS)/usr/lib/arm-linux-gnueabihf
+          GLLINK += -L$(TARGET_FS)/usr/lib/arm-linux-gnueabihf
+        endif
+      else ifeq ($(HOST_ARCH)-$(TARGET_ARCH),x86_64-aarch64)
+        GLPATH := /usr/aarch64-linux-gnu/lib
+        GLLINK := -L/usr/aarch64-linux-gnu/lib
+        ifneq ($(TARGET_FS),)
+          GLPATH += $(TARGET_FS)/usr/lib
+          GLPATH += $(TARGET_FS)/usr/lib/aarch64-linux-gnu
+          GLLINK += -L$(TARGET_FS)/usr/lib/aarch64-linux-gnu
+        endif 
+      else
+        UBUNTU_PKG_NAME = $(shell which dpkg >/dev/null 2>&1 && dpkg -l 'nvidia-*' | grep '^ii' | awk '{print $$2}' | head -1)
+        ifneq ("$(UBUNTU_PKG_NAME)","")
+          GLPATH    ?= /usr/lib/$(UBUNTU_PKG_NAME)
+          GLLINK    ?= -L/usr/lib/$(UBUNTU_PKG_NAME)
+        endif
+
+        DFLT_PATH ?= /usr/lib
+      endif
+    endif
+    ifeq ("$(SUSE)","0")
+      GLPATH    ?= /usr/X11R6/lib64
+      GLLINK    ?= -L/usr/X11R6/lib64
+      DFLT_PATH ?= /usr/lib64
+    endif
+    ifeq ("$(FEDORA)","0")
+      GLPATH    ?= /usr/lib64/nvidia
+      GLLINK    ?= -L/usr/lib64/nvidia
+      DFLT_PATH ?= /usr/lib64
+    endif
+    ifeq ("$(RHEL)","0")
+      GLPATH    ?= /usr/lib64/nvidia
+      GLLINK    ?= -L/usr/lib64/nvidia
+      DFLT_PATH ?= /usr/lib64
+    endif
+    ifeq ("$(CENTOS)","0")
+      GLPATH    ?= /usr/lib64/nvidia
+      GLLINK    ?= -L/usr/lib64/nvidia
+      DFLT_PATH ?= /usr/lib64
+    endif
+
+  EGLLIB  := $(shell find -L $(GLPATH) $(DFLT_PATH) -name libEGL.so    -print 2>/dev/null)
+
+  ifeq ("$(EGLLIB)","")
+      $(info >>> WARNING - libEGL.so not found, please install libEGL.so <<<)
+      SAMPLE_ENABLED := 0
+  endif
+
+  HEADER_SEARCH_PATH ?= $(TARGET_FS)/usr/include
+  ifeq ($(HOST_ARCH)-$(TARGET_ARCH)-$(TARGET_OS),x86_64-armv7l-linux)
+      HEADER_SEARCH_PATH += /usr/arm-linux-gnueabihf/include
+  else ifeq ($(HOST_ARCH)-$(TARGET_ARCH)-$(TARGET_OS),x86_64-aarch64-linux)
+      HEADER_SEARCH_PATH += /usr/aarch64-linux-gnu/include
+  endif
+
+  EGLHEADER  := $(shell find -L $(HEADER_SEARCH_PATH) -name egl.h -print 2>/dev/null)
+  EGLEXTHEADER  := $(shell find -L $(HEADER_SEARCH_PATH) -name eglext.h -print 2>/dev/null)
+
+  ifeq ("$(EGLHEADER)","")
+      $(info >>> WARNING - egl.h not found, please install egl.h <<<)
+      SAMPLE_ENABLED := 0
+  endif
+  ifeq ("$(EGLEXTHEADER)","")
+      $(info >>> WARNING - eglext.h not found, please install eglext.h <<<)
+      SAMPLE_ENABLED := 0
+  endif
+else
+endif
+
+ifeq ("$(TARGET_OS)","qnx")
+    HOST_CCFLAGS := -V5.4.0,gcc_ntoaarch64le
+endif
+
+# Attempt to compile a minimal EGL application and run to check if EGL_SUPPORT_REUSE_NV is supported in the EGL headers available.
+ifneq ($(SAMPLE_ENABLED), 0)
+      $(shell printf "#include <EGL/egl.h>\n#include <EGL/eglext.h>\nint main() {\n#ifdef EGL_SUPPORT_REUSE_NV \n #error \"Compatible EGL header found\" \n  return 0;\n#endif \n return 1;\n}"  > test.c; )
+      EGL_DEFINES := $(shell $(HOST_COMPILER) $(HOST_CCFLAGS) $(CCFLAGS) $(EXTRA_CCFLAGS) -lEGL test.c -c 2>&1 | grep -ic "Compatible EGL header found";)
+      SHOULD_WAIVE := 0
+      ifeq ($(EGL_DEFINES),0)
+        SHOULD_WAIVE := 1
+      endif
+      ifeq ($(SHOULD_WAIVE),1)
+          $(info -----------------------------------------------------------------------------------------------)
+          $(info WARNING - NVIDIA EGL EXTENSIONS are not available in the present EGL headers)
+          $(info -----------------------------------------------------------------------------------------------)
+          $(info   This CUDA Sample cannot be built if the EGL NVIDIA EXTENSIONS like EGL_SUPPORT_REUSE_NV are not supported in EGL headers.)
+          $(info   This will be a dry-run of the Makefile.)
+          $(info   Please install the latest khronos EGL headers and libs to build this sample)
+          $(info -----------------------------------------------------------------------------------------------)
+          SAMPLE_ENABLED := 0
+      endif
+      $(shell rm test.o test.c 2>/dev/null)
+endif
+

Samples/EGLStreams_CUDA_Interop/main.cpp

@@ -0,0 +1,231 @@
+/* Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+//
+// DESCRIPTION:   Simple EGL stream sample app
+//
+//
+
+//#define EGL_EGLEXT_PROTOTYPES
+
+#include "cudaEGL.h"
+#include "cuda_consumer.h"
+#include "cuda_producer.h"
+#include "eglstrm_common.h"
+
+/* ------  globals ---------*/
+
+#if defined(EXTENSION_LIST)
+EXTENSION_LIST(EXTLST_EXTERN)
+#endif
+
+#define NUM_TRAILS 4
+
+bool signal_stop = 0;
+
+static void sig_handler(int sig) {
+  signal_stop = 1;
+  printf("Signal: %d\n", sig);
+}
+
+int main(int argc, char **argv) {
+  TestArgs args;
+  CUresult curesult = CUDA_SUCCESS;
+  unsigned int i, j;
+  EGLint streamState = 0;
+
+  test_cuda_consumer_s cudaConsumer;
+  test_cuda_producer_s cudaProducer;
+
+  memset(&cudaProducer, 0, sizeof(test_cuda_producer_s));
+  memset(&cudaConsumer, 0, sizeof(test_cuda_consumer_s));
+
+  // Hook up Ctrl-C handler
+  signal(SIGINT, sig_handler);
+  if (!eglSetupExtensions()) {
+    printf("SetupExtentions failed \n");
+    curesult = CUDA_ERROR_UNKNOWN;
+    goto done;
+  }
+
+  checkCudaErrors(cuInit(0));
+
+  int count;
+
+  checkCudaErrors(cuDeviceGetCount(&count));
+  printf("Found %d cuda devices\n", count);
+
+  CUdevice devId;
+
+  if (!EGLStreamInit(&devId)) {
+    printf("EGLStream Init failed.\n");
+    curesult = CUDA_ERROR_UNKNOWN;
+    goto done;
+  }
+  curesult = cudaDeviceCreateProducer(&cudaProducer, devId);
+  if (curesult != CUDA_SUCCESS) {
+    goto done;
+  }
+  curesult = cudaDeviceCreateConsumer(&cudaConsumer, devId);
+  if (curesult != CUDA_SUCCESS) {
+    goto done;
+  }
+  checkCudaErrors(cuCtxPushCurrent(cudaConsumer.context));
+  if (CUDA_SUCCESS != (curesult = cuEGLStreamConsumerConnect(
+                           &(cudaConsumer.cudaConn), eglStream))) {
+    printf("FAILED Connect CUDA consumer  with error %d\n", curesult);
+    goto done;
+  } else {
+    printf("Connected CUDA consumer, CudaConsumer %p\n", cudaConsumer.cudaConn);
+  }
+  checkCudaErrors(cuCtxPopCurrent(&cudaConsumer.context));
+
+  checkCudaErrors(cuCtxPushCurrent(cudaProducer.context));
+  if (CUDA_SUCCESS ==
+      (curesult = cuEGLStreamProducerConnect(&(cudaProducer.cudaConn),
+                                             eglStream, WIDTH, HEIGHT))) {
+    printf("Connect CUDA producer Done, CudaProducer %p\n",
+           cudaProducer.cudaConn);
+  } else {
+    printf("Connect CUDA producer FAILED with error %d\n", curesult);
+    goto done;
+  }
+  checkCudaErrors(cuCtxPopCurrent(&cudaProducer.context));
+
+  // Initialize producer
+  for (i = 0; i < NUM_TRAILS; i++) {
+    if (streamState != EGL_STREAM_STATE_CONNECTING_KHR) {
+      if (!eglQueryStreamKHR(g_display, eglStream, EGL_STREAM_STATE_KHR,
+                             &streamState)) {
+        printf("main: eglQueryStreamKHR EGL_STREAM_STATE_KHR failed\n");
+        curesult = CUDA_ERROR_UNKNOWN;
+        goto done;
+      }
+    }
+    args.inputWidth = WIDTH;
+    args.inputHeight = HEIGHT;
+    if (i % 2 != 0) {
+      args.isARGB = 1;
+      args.infile1 = sdkFindFilePath("cuda_f_1.yuv", argv[0]);
+      args.infile2 = sdkFindFilePath("cuda_f_2.yuv", argv[0]);
+    } else {
+      args.isARGB = 0;
+      args.infile1 = sdkFindFilePath("cuda_yuv_f_1.yuv", argv[0]);
+      args.infile2 = sdkFindFilePath("cuda_yuv_f_2.yuv", argv[0]);
+    }
+    if ((i % 4) < 2) {
+      args.pitchLinearOutput = 1;
+    } else {
+      args.pitchLinearOutput = 0;
+    }
+
+    checkCudaErrors(cuCtxPushCurrent(cudaProducer.context));
+    cudaProducerInit(&cudaProducer, g_display, eglStream, &args);
+    checkCudaErrors(cuCtxPopCurrent(&cudaProducer.context));
+
+    checkCudaErrors(cuCtxPushCurrent(cudaConsumer.context));
+    cuda_consumer_init(&cudaConsumer, &args);
+    checkCudaErrors(cuCtxPopCurrent(&cudaConsumer.context));
+
+    printf("main - Cuda Producer and Consumer Initialized.\n");
+
+    for (j = 0; j < 2; j++) {
+      printf("Running for %s frame and %s input\n",
+             args.isARGB ? "ARGB" : "YUV",
+             args.pitchLinearOutput ? "Pitchlinear" : "BlockLinear");
+      if (j == 0) {
+        checkCudaErrors(cuCtxPushCurrent(cudaProducer.context));
+        curesult = cudaProducerTest(&cudaProducer, cudaProducer.fileName1);
+        if (curesult != CUDA_SUCCESS) {
+          printf("Cuda Producer Test failed for frame = %d\n", j + 1);
+          goto done;
+        }
+        checkCudaErrors(cuCtxPopCurrent(&cudaProducer.context));
+        checkCudaErrors(cuCtxPushCurrent(cudaConsumer.context));
+        curesult = cudaConsumerTest(&cudaConsumer, cudaConsumer.outFile1);
+        if (curesult != CUDA_SUCCESS) {
+          printf("Cuda Consumer Test failed for frame = %d\n", j + 1);
+          goto done;
+        }
+        checkCudaErrors(cuCtxPopCurrent(&cudaConsumer.context));
+      } else {
+        checkCudaErrors(cuCtxPushCurrent(cudaProducer.context));
+        curesult = cudaProducerTest(&cudaProducer, cudaProducer.fileName2);
+        if (curesult != CUDA_SUCCESS) {
+          printf("Cuda Producer Test failed for frame = %d\n", j + 1);
+          goto done;
+        }
+
+        checkCudaErrors(cuCtxPopCurrent(&cudaProducer.context));
+        checkCudaErrors(cuCtxPushCurrent(cudaConsumer.context));
+        curesult = cudaConsumerTest(&cudaConsumer, cudaConsumer.outFile2);
+        if (curesult != CUDA_SUCCESS) {
+          printf("Cuda Consumer Test failed for frame = %d\n", j + 1);
+          goto done;
+        }
+        checkCudaErrors(cuCtxPopCurrent(&cudaConsumer.context));
+      }
+    }
+  }
+
+  checkCudaErrors(cuCtxPushCurrent(cudaProducer.context));
+  if (CUDA_SUCCESS != (curesult = cudaProducerDeinit(&cudaProducer))) {
+    printf("Producer Disconnect FAILED. \n");
+    goto done;
+  }
+  checkCudaErrors(cuCtxPopCurrent(&cudaProducer.context));
+
+  if (!eglQueryStreamKHR(g_display, eglStream, EGL_STREAM_STATE_KHR,
+                         &streamState)) {
+    printf("Cuda consumer, eglQueryStreamKHR EGL_STREAM_STATE_KHR failed\n");
+    curesult = CUDA_ERROR_UNKNOWN;
+    goto done;
+  }
+  if (streamState != EGL_STREAM_STATE_DISCONNECTED_KHR) {
+    if (CUDA_SUCCESS != (curesult = cuda_consumer_deinit(&cudaConsumer))) {
+      printf("Consumer Disconnect FAILED.\n");
+      goto done;
+    }
+  }
+  printf("Producer and Consumer Disconnected \n");
+
+done:
+  if (!eglQueryStreamKHR(g_display, eglStream, EGL_STREAM_STATE_KHR,
+                         &streamState)) {
+    printf("Cuda consumer, eglQueryStreamKHR EGL_STREAM_STATE_KHR failed\n");
+    curesult = CUDA_ERROR_UNKNOWN;
+  }
+  if (streamState != EGL_STREAM_STATE_DISCONNECTED_KHR) {
+    EGLStreamFini();
+  }
+
+  if (curesult == CUDA_SUCCESS) {
+    printf("&&&& EGLStream interop test PASSED\n");
+  } else {
+    printf("&&&& EGLStream interop test FAILED\n");
+  }
+  return 0;
+}

Samples/EGLSync_CUDAEvent_Interop/.vscode/c_cpp_properties.json

@@ -0,0 +1,18 @@
+{
+    "configurations": [
+        {
+            "name": "Linux",
+            "includePath": [
+                "${workspaceFolder}/**",
+                "${workspaceFolder}/../../Common"
+            ],
+            "defines": [],
+            "compilerPath": "/usr/local/cuda/bin/nvcc",
+            "cStandard": "gnu17",
+            "cppStandard": "gnu++14",
+            "intelliSenseMode": "linux-gcc-x64",
+            "configurationProvider": "ms-vscode.makefile-tools"
+        }
+    ],
+    "version": 4
+}

Samples/EGLSync_CUDAEvent_Interop/.vscode/extensions.json

@@ -0,0 +1,7 @@
+{
+    "recommendations": [
+        "nvidia.nsight-vscode-edition",
+        "ms-vscode.cpptools",
+        "ms-vscode.makefile-tools"
+    ]
+}

Samples/EGLSync_CUDAEvent_Interop/.vscode/launch.json

@@ -0,0 +1,10 @@
+{
+    "configurations": [
+        {
+            "name": "CUDA C++: Launch",
+            "type": "cuda-gdb",
+            "request": "launch",
+            "program": "${workspaceFolder}/EGLSync_CUDAEvent_Interop"
+        }
+    ]
+}

Samples/EGLSync_CUDAEvent_Interop/.vscode/tasks.json

@@ -0,0 +1,15 @@
+{
+    "version": "2.0.0",
+    "tasks": [
+        {
+            "label": "sample",
+            "type": "shell",
+            "command": "make dbg=1",
+            "problemMatcher": ["$nvcc"],
+            "group": {
+                "kind": "build",
+                "isDefault": true
+            }
+        }
+    ]
+}

Samples/EGLSync_CUDAEvent_Interop/EGLSync_CUDAEvent_Interop.cu

@@ -0,0 +1,783 @@
+/* Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+// Simple interop app demonstrating EGLImage + EGLSync interop with CUDA.
+// Using EGLSync - CUDA Event interop one can achieve synchronization on GPU
+// itself for GL-EGL-CUDA operations instead of blocking CPU for
+// synchronization. This app requires GLES 3.2 or higher
+
+//---------------------------INCLUDES---------------------------------//
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <time.h>
+#include "graphics_interface.h"
+#include <cuda.h>
+#include <helper_cuda_drvapi.h>
+#include <cudaEGL.h>
+#include <EGL/egl.h>
+#include <EGL/eglext.h>
+#include <GLES3/gl32.h>
+#include "egl_common.h"
+
+//---------------------------DEFINES---------------------------------//
+#define MAX_ITR 100
+
+#define FAILURE 0
+#define SUCCESS 1
+#define WAIVED 2
+
+#define BLOCK_SIZE 16
+
+#define GL_READ 0
+#define GL_WRITE 1
+//---------------------------MACROS---------------------------------//
+
+// Error-checking wrapper around GL calls
+#define GL_SAFE_CALL(call)                                              \
+  {                                                                     \
+    GLenum err;                                                         \
+    call;                                                               \
+    err = glGetError();                                                 \
+    if (err != GL_NO_ERROR) {                                           \
+      fprintf(stderr, "%s:%d GL error: %d\n", __FILE__, __LINE__, err); \
+      cleanup(FAILURE);                                                 \
+    }                                                                   \
+  }
+
+#define GL_SAFE_CALL_NO_CLEANUP(call, err)                                 \
+  {                                                                        \
+    GLenum status;                                                         \
+    call;                                                                  \
+    status = glGetError();                                                 \
+    if (status != GL_NO_ERROR) {                                           \
+      fprintf(stderr, "%s:%d GL error: %d\n", __FILE__, __LINE__, status); \
+      err = status;                                                        \
+    }                                                                      \
+  }
+
+// Error-checking wrapper around CUDA calls (taken from cutil.h)
+#define CUDA_SAFE_CALL(call)                                                  \
+  do {                                                                        \
+    cudaError err = call;                                                     \
+    if (cudaSuccess != err) {                                                 \
+      fprintf(stderr, "Cuda error in file '%s' in line %i : %s.\n", __FILE__, \
+              __LINE__, cudaGetErrorString(err));                             \
+      cleanup(FAILURE);                                                       \
+    }                                                                         \
+  } while (0)
+
+#define CUDA_SAFE_CALL_NO_CLEANUP(call, err)                                  \
+  do {                                                                        \
+    cudaError status = call;                                                  \
+    if (cudaSuccess != status) {                                              \
+      fprintf(stderr, "Cuda error in file '%s' in line %i : %s.\n", __FILE__, \
+              __LINE__, cudaGetErrorString(status));                          \
+      err = status;                                                           \
+    }                                                                         \
+  } while (0)
+
+#if defined(EXTENSION_LIST)
+EXTENSION_LIST(EXTLST_DECL)
+typedef void (*extlst_fnptr_t)(void);
+static struct {
+  extlst_fnptr_t *fnptr;
+  char const *name;
+} extensionList[] = {EXTENSION_LIST(EXTLST_ENTRY)};
+
+int eglSetupExtensions(void) {
+  unsigned int i;
+
+  for (i = 0; i < (sizeof(extensionList) / sizeof(*extensionList)); i++) {
+    *extensionList[i].fnptr = eglGetProcAddress(extensionList[i].name);
+    if (*extensionList[i].fnptr == NULL) {
+      printf("Couldn't get address of %s()\n", extensionList[i].name);
+      return 0;
+    }
+  }
+
+  return 1;
+}
+#endif
+
+#if defined(EXTENSION_LIST)
+EXTENSION_LIST(EXTLST_EXTERN)
+#endif
+
+//------------------------GLOBAL VARIABLES--------------------------//
+
+// GL texture
+GLuint tex[2] = {0};
+
+// Used to catch unexpected termination from GLUT
+int cleanExit = 0;
+
+// Use CPU Sync or GPU sync; Default GPU
+int useGpu = 1;
+
+// CUDA Resource
+CUgraphicsResource writeResource = NULL;
+CUgraphicsResource readResource = NULL;
+CUarray writeArray, readArray;
+CUdevice device;
+CUcontext context;
+
+// Which device to run on
+unsigned int dev = 0;
+
+// Default width, height, and iterations value
+int width = 2048;
+int height = 2048;
+int itr = MAX_ITR;
+
+// Error check variable
+__device__ static unsigned int numErrors = 0;
+
+//-----------------------FUNCTION PROTOTYPES------------------------//
+
+void checkSync(int argc, char **argv);
+int parseCmdLine(int argc, char **argv);
+void printUsage(void);
+void cleanup(int status);
+void exitHandler(void);
+void printStatus(int status);
+void checkSyncOnCPU(void);
+void checkSyncOnGPU(EGLDisplay dpy);
+
+__global__ void verify_and_update_kernel(CUsurfObject write, CUsurfObject read,
+                                         char expected, char newval, int width,
+                                         int height);
+extern "C" cudaError_t cudaGetValueMismatch();
+
+//-----------------------FUNCTION DEFINITIONS------------------------//
+
+int main(int argc, char *argv[]) {
+#if defined(__linux__)
+  setenv("DISPLAY", ":0", 0);
+#endif
+
+  parseCmdLine(argc, argv);
+  atexit(exitHandler);
+
+  checkSync(argc, argv);
+  return 0;
+}
+
+int parseCmdLine(int argc, char **argv) {
+  int i;
+  for (i = 1; i < argc; i++) {
+    if (strcmp(argv[i], "-cpu") == 0) {
+      useGpu = 0;
+    }
+
+    if (strcmp(argv[i], "-h") == 0) {
+      printUsage();
+      cleanup(SUCCESS);
+    }
+
+    if (strcmp(argv[i], "-width") == 0) {
+      ++i;
+      if (i == argc) {
+        printf("width option must be followed by value\n");
+        return FAILURE;
+      }
+      if (sscanf(argv[i], "%d", &width) != 1) {
+        printf("Error: invalid width value\n");
+        return FAILURE;
+      }
+    }
+
+    if (strcmp(argv[i], "-height") == 0) {
+      ++i;
+      if (i == argc) {
+        printf("height option must be followed by value\n");
+        return FAILURE;
+      }
+      if (sscanf(argv[i], "%d", &height) != 1) {
+        printf("Error: invalid height value\n");
+        return FAILURE;
+      }
+    }
+    if (strcmp(argv[i], "-itr") == 0) {
+      ++i;
+      if (i == argc) {
+        printf("itr option must be followed by iteration value\n");
+        return FAILURE;
+      }
+      if (sscanf(argv[i], "%d", &itr) != 1) {
+        printf("Error: invalid iteration value\n");
+        return FAILURE;
+      }
+    }
+  }
+
+  return SUCCESS;
+}
+
+void printUsage(void) {
+  printf("Usage:\n");
+  printf("\t-h\tPrint command line options\n");
+  printf("\t-cpu\tSync on the CPU instead of the GPU\n");
+  printf("\t-width w\tSet the width to w\n");
+  printf("\t-height h\tSet the height to h\n");
+  printf("\t-itr i\tSet number of iterations to i\n");
+}
+
+void checkSync(int argc, char **argv) {
+  int x, y;
+  int bufferSize = width * height * 4;
+  unsigned char *pSurf_read = NULL, *pSurf_write = NULL;
+  int integrated;
+
+  CUresult status = CUDA_SUCCESS;
+
+  // Init values for variables
+  x = y = 0;
+
+  if (CUDA_SUCCESS != (status = cuInit(0))) {
+    printf("Failed to initialize CUDA\n");
+  }
+  device = findCudaDeviceDRV(argc, (const char **)argv);
+
+  if (CUDA_SUCCESS != (status = cuCtxCreate(&context, 0, device))) {
+    printf("failed to create CUDA context\n");
+  }
+  cuCtxPushCurrent(context);
+
+  status =
+      cuDeviceGetAttribute(&integrated, CU_DEVICE_ATTRIBUTE_INTEGRATED, device);
+  if (status != CUDA_SUCCESS) {
+    printf("Failed to get device attribute CU_DEVICE_ATTRIBUTE_INTEGRATED\n");
+    cleanup(FAILURE);
+  }
+
+  if (integrated != 1) {
+    printf(
+        "EGLSync_CUDAEvent_Interop does not support dGPU. Waiving sample.\n");
+    cleanup(WAIVED);
+  }
+
+#if (defined(__arm__) || defined(__aarch64__)) && defined(__linux__)
+  graphics_setup_window(0, 0, width, height, "EGLSync_CUDA_Interop");
+#endif
+
+  pSurf_read = (unsigned char *)malloc(bufferSize);
+  pSurf_write = (unsigned char *)malloc(bufferSize);
+  if (pSurf_read == NULL || pSurf_write == NULL) {
+    printf("malloc failed\n");
+    cleanup(FAILURE);
+  }
+
+  for (x = 0; x < width; x++) {
+    for (y = 0; y < height; y++) {
+      pSurf_read[(y * width + x) * 4] = 1;
+      pSurf_read[(y * width + x) * 4 + 1] = 1;
+      pSurf_read[(y * width + x) * 4 + 2] = 1;
+      pSurf_read[(y * width + x) * 4 + 3] = 1;
+      pSurf_write[(y * width + x) * 4] = 0;
+      pSurf_write[(y * width + x) * 4 + 1] = 0;
+      pSurf_write[(y * width + x) * 4 + 2] = 0;
+      pSurf_write[(y * width + x) * 4 + 3] = 0;
+    }
+  }
+
+  // NOP call to error-check the above glut calls
+  GL_SAFE_CALL({});
+
+  // Init texture
+  GL_SAFE_CALL(glGenTextures(2, tex));
+
+  GL_SAFE_CALL(glBindTexture(GL_TEXTURE_2D, tex[GL_READ]));
+  GL_SAFE_CALL(
+      glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST));
+  GL_SAFE_CALL(
+      glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST));
+  GL_SAFE_CALL(glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA8, width, height, 0,
+                            GL_RGBA, GL_UNSIGNED_BYTE, pSurf_read));
+  GL_SAFE_CALL(glBindTexture(GL_TEXTURE_2D, 0));
+
+  GL_SAFE_CALL(glBindTexture(GL_TEXTURE_2D, tex[GL_WRITE]));
+  GL_SAFE_CALL(
+      glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST));
+  GL_SAFE_CALL(
+      glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST));
+  GL_SAFE_CALL(glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA8, width, height, 0,
+                            GL_RGBA, GL_UNSIGNED_BYTE, pSurf_write));
+  GL_SAFE_CALL(glBindTexture(GL_TEXTURE_2D, 0));
+
+  glFinish();
+
+  EGLDisplay eglDisplayHandle = eglGetCurrentDisplay();
+  if (eglDisplayHandle == EGL_NO_DISPLAY) {
+    printf("eglDisplayHandle failed \n");
+    cleanup(FAILURE);
+  } else {
+    printf("eglDisplay Handle created \n");
+  }
+
+  if (!eglSetupExtensions()) {
+    printf("SetupExtentions failed \n");
+    cleanup(FAILURE);
+  }
+
+  EGLContext eglCtx = eglGetCurrentContext();
+  if (eglCtx == EGL_NO_CONTEXT) {
+    printf("Context1 create failed with error %d\n", eglGetError());
+    cleanup(FAILURE);
+  }
+
+  // Create the EGL_Image
+  EGLint eglImgAttrs[] = {EGL_IMAGE_PRESERVED_KHR, EGL_TRUE, EGL_NONE,
+                          EGL_NONE};
+
+  EGLImageKHR eglImage1 =
+      eglCreateImageKHR(eglDisplayHandle, eglCtx, EGL_GL_TEXTURE_2D_KHR,
+                        (EGLClientBuffer)(intptr_t)tex[GL_READ], eglImgAttrs);
+  if (eglImage1 == EGL_NO_IMAGE_KHR) {
+    printf("EGLImage create failed for read texture with error %d\n",
+           eglGetError());
+    cleanup(FAILURE);
+  } else {
+    printf("EGLImage1 created \n");
+  }
+
+  EGLImageKHR eglImage2 =
+      eglCreateImageKHR(eglDisplayHandle, eglCtx, EGL_GL_TEXTURE_2D_KHR,
+                        (EGLClientBuffer)(intptr_t)tex[GL_WRITE], eglImgAttrs);
+  if (eglImage2 == EGL_NO_IMAGE_KHR) {
+    printf("EGLImage create failed for write texture with error %d\n",
+           eglGetError());
+    cleanup(FAILURE);
+  } else {
+    printf("EGLImage2 created \n");
+  }
+
+  glFinish();
+
+  status = cuGraphicsEGLRegisterImage(&writeResource, eglImage1,
+                                      CU_GRAPHICS_REGISTER_FLAGS_NONE);
+  if (status != CUDA_SUCCESS) {
+    printf("cuGraphicsEGLRegisterImage failed with Texture 1\n");
+    cleanup(FAILURE);
+  } else {
+    printf(
+        "cuGraphicsEGLRegisterImage Passed, writeResource created with texture "
+        "1\n");
+  }
+
+  status =
+      cuGraphicsSubResourceGetMappedArray(&writeArray, writeResource, 0, 0);
+  if (status != CUDA_SUCCESS) {
+    printf(
+        "cuGraphicsSubResourceGetMappedArray failed for writeResource with "
+        "texture 1\n");
+    cleanup(FAILURE);
+  }
+
+  status = cuGraphicsEGLRegisterImage(&readResource, eglImage2,
+                                      CU_GRAPHICS_REGISTER_FLAGS_NONE);
+  if (status != CUDA_SUCCESS) {
+    printf(
+        "cuGraphicsEGLRegisterImage failed for readResource with Texture 2\n");
+    cleanup(FAILURE);
+  } else {
+    printf(
+        "cuGraphicsEGLRegisterImage Passed, readResource created with texture "
+        "2\n");
+  }
+
+  status = cuGraphicsSubResourceGetMappedArray(&readArray, readResource, 0, 0);
+  if (status != CUDA_SUCCESS) {
+    printf("cuGraphicsSubResourceGetMappedArray failed for texture 2\n");
+    cleanup(FAILURE);
+  }
+
+  if (useGpu) {
+    printf("Using GPU Sync path\n");
+    checkSyncOnGPU(eglDisplayHandle);
+  } else {
+    printf("Using CPU Sync path\n");
+    checkSyncOnCPU();
+  }
+
+  free(pSurf_read);
+  free(pSurf_write);
+  cleanup(SUCCESS);
+}
+
+void checkSyncOnCPU(void) {
+  int z = 0;
+  unsigned char expectedData, newData;
+  CUresult status = CUDA_SUCCESS;
+  CUDA_RESOURCE_DESC wdsc, rdsc;
+  memset(&wdsc, 0, sizeof(wdsc));
+  memset(&rdsc, 0, sizeof(rdsc));
+
+  expectedData = 0;
+  newData = 1;
+
+  wdsc.resType = CU_RESOURCE_TYPE_ARRAY;
+  wdsc.res.array.hArray = writeArray;
+  CUsurfObject writeSurface;
+  rdsc.resType = CU_RESOURCE_TYPE_ARRAY;
+  rdsc.res.array.hArray = readArray;
+  CUsurfObject readSurface;
+
+  status = cuSurfObjectCreate(&writeSurface, &wdsc);
+  if (status != CUDA_SUCCESS) {
+    printf("Surface bounding failed with status %d\n", status);
+    cleanup(FAILURE);
+  }
+  status = cuSurfObjectCreate(&readSurface, &rdsc);
+  if (status != CUDA_SUCCESS) {
+    printf("Surface bounding failed\n");
+    cleanup(FAILURE);
+  }
+
+  for (z = 0; z < itr; z++) {
+    // GL call to copy from read texture to write texture
+    GL_SAFE_CALL(glCopyImageSubData(tex[GL_READ], GL_TEXTURE_2D, 0, 0, 0, 0,
+                                    tex[GL_WRITE], GL_TEXTURE_2D, 0, 0, 0, 0,
+                                    width, height, 1));
+
+    glFinish();
+
+    newData++;
+    expectedData++;
+
+    verify_and_update_kernel<<<(width * height) / 256, 256>>>(
+        writeSurface, readSurface, expectedData, newData, width, height);
+
+    status = cuCtxSynchronize();
+    if (status != CUDA_SUCCESS) {
+      printf("cuCtxSynchronize failed \n");
+    }
+  }
+
+  cudaError_t err = cudaGetValueMismatch();
+  if (err != cudaSuccess) {
+    printf("Value mismatch seen when using CPU sync\n");
+    cleanup(FAILURE);
+  }
+
+  // Clean up CUDA writeResource
+  status = cuGraphicsUnregisterResource(writeResource);
+  if (status != CUDA_SUCCESS) {
+    printf("Failed to unregister %d", status);
+    cleanup(FAILURE);
+  } else {
+    printf("Unregistered writeResource. \n");
+  }
+
+  // Clean up CUDA readResource
+  status = cuGraphicsUnregisterResource(readResource);
+  if (status != CUDA_SUCCESS) {
+    printf("Failed to unregister %d", status);
+    cleanup(FAILURE);
+  } else {
+    printf("Unregistered readResource. \n");
+  }
+}
+
+/*
+    Performs same function as checkSyncOnCPU
+    Here instead of glFinish() and cuCtxSynchronize like in checkSyncOnCPU,
+    we make use of EGLSync, CUDA Event and cuStreamWaitEvent, eglWaitSyncKHR to
+   achieve the synchronization due to this CPU is not blocked for any
+   synchronization needed between GL-EGL & CUDA operations all synchronizations
+   happens on the GPU only.
+*/
+void checkSyncOnGPU(EGLDisplay dpy) {
+  int z = 0;
+  unsigned char expectedData, newData;
+  cudaError_t err;
+  CUresult status = CUDA_SUCCESS;
+  CUstream stream;
+  CUevent timingDisabledEvent;
+  CUDA_RESOURCE_DESC wdsc, rdsc;
+  memset(&wdsc, 0, sizeof(wdsc));
+  memset(&rdsc, 0, sizeof(rdsc));
+
+  expectedData = 0;
+  newData = 1;
+
+  wdsc.resType = CU_RESOURCE_TYPE_ARRAY;
+  wdsc.res.array.hArray = writeArray;
+  CUsurfObject writeSurface;
+  rdsc.resType = CU_RESOURCE_TYPE_ARRAY;
+  rdsc.res.array.hArray = readArray;
+  CUsurfObject readSurface;
+
+  status = cuSurfObjectCreate(&writeSurface, &wdsc);
+  if (status != CUDA_SUCCESS) {
+    printf("Surface bounding failed with status %d\n", status);
+    cleanup(FAILURE);
+  }
+  status = cuSurfObjectCreate(&readSurface, &rdsc);
+  if (status != CUDA_SUCCESS) {
+    printf("Surface bounding failed\n");
+    cleanup(FAILURE);
+  }
+
+  status = cuStreamCreate(&stream, CU_STREAM_DEFAULT);
+  if (status != CUDA_SUCCESS) {
+    printf("Stream creation failed\n");
+    cleanup(FAILURE);
+  }
+
+  // Creates timing disabled event which uses non-blocking synchronization
+  status = cuEventCreate(&timingDisabledEvent, CU_EVENT_DISABLE_TIMING);
+  if (status != CUDA_SUCCESS) {
+    printf("Default event creation failed\n");
+    cleanup(FAILURE);
+  }
+
+  /*
+      1. We perform texture-to-texture copy in GLES which is async function
+      2. Followed by creating EGLSync and a CUDA Event from that EGLSync object
+      3. Using cuStreamWaitEvent() we wait in GPU for the GLES to finish texture
+     copy.
+      4. CUDA kernel verfiy_and_update_kernel verifies if the copied data by
+     GLES is correct, and it updates the buffer with new values.
+      5. This is followed by eglWaitSyncKHR() which waits for the cuda kernel to
+     finish, so that in the next iteration GLES can perform the copying of the
+     updated buffer to write texture,
+  */
+  for (z = 0; z < itr; z++) {
+    // GL call to copy from read texture to write texture
+    GL_SAFE_CALL(glCopyImageSubData(tex[GL_READ], GL_TEXTURE_2D, 0, 0, 0, 0,
+                                    tex[GL_WRITE], GL_TEXTURE_2D, 0, 0, 0, 0,
+                                    width, height, 1));
+
+    EGLSyncKHR eglSyncForGL, eglSyncForCuda;
+    EGLBoolean egl_status = EGL_TRUE;
+    EGLAttribKHR eglattrib[] = {EGL_CUDA_EVENT_HANDLE_NV,
+                                (EGLAttrib)timingDisabledEvent, EGL_NONE};
+
+    CUevent cudaEGLSyncEvent;
+
+    eglSyncForGL = eglCreateSyncKHR(dpy, EGL_SYNC_FENCE_KHR, NULL);
+
+    if (eglSyncForGL == EGL_NO_SYNC_KHR) {
+      printf(" EGL Sync creation failed\n");
+      cleanup(FAILURE);
+    }
+
+    status = cuEventCreateFromEGLSync(&cudaEGLSyncEvent, eglSyncForGL,
+                                      CU_EVENT_DEFAULT);
+    if (status != CUDA_SUCCESS) {
+      printf("CUDA event creation from EGLSync failed\n");
+      cleanup(FAILURE);
+    }
+
+    // We wait from CUDA in GPU for GL-EGL operation completion
+    status = cuStreamWaitEvent(stream, cudaEGLSyncEvent, 0);
+    if (status != CUDA_SUCCESS) {
+      printf("Stream wait for event created from EGLSync failed\n");
+      cleanup(FAILURE);
+    }
+
+    egl_status = eglDestroySyncKHR(dpy, eglSyncForGL);
+    if (egl_status != EGL_TRUE) {
+      printf("EGL sync object destruction failed\n");
+      cleanup(FAILURE);
+    }
+
+    newData++;
+    expectedData++;
+
+    // Verifies the values in readSurface which is copied by
+    // glCopyImageSubData() And writes value of newData into writeSurface
+    verify_and_update_kernel<<<(width * height) / 256, 256, 0, stream>>>(
+        writeSurface, readSurface, expectedData, newData, width, height);
+
+    status = cuEventDestroy(cudaEGLSyncEvent);
+    if (status != CUDA_SUCCESS) {
+      printf("Event Destroy failed\n");
+      cleanup(FAILURE);
+    }
+
+    status = cuEventRecord(timingDisabledEvent, stream);
+    if (status != CUDA_SUCCESS) {
+      printf("Event Record failed\n");
+      cleanup(FAILURE);
+    }
+
+    // creating an EGL sync object linked to a CUDA event object
+    eglSyncForCuda = eglCreateSync64KHR(dpy, EGL_SYNC_CUDA_EVENT_NV, eglattrib);
+
+    // We wait from EGL for CUDA operation completion
+    egl_status = eglWaitSyncKHR(dpy, eglSyncForCuda, 0);
+    if (egl_status != EGL_TRUE) {
+      printf("eglWaitSyncKHR failed\n");
+      cleanup(FAILURE);
+    }
+    egl_status = eglDestroySyncKHR(dpy, eglSyncForCuda);
+    if (egl_status != EGL_TRUE) {
+      printf("EGL sync object destruction failed\n");
+      cleanup(FAILURE);
+    }
+  }
+
+  err = cudaGetValueMismatch();
+  if (err != cudaSuccess) {
+    printf("Value mismatch seen when using GPU sync\n");
+    cleanup(FAILURE);
+  }
+
+  // Clean up CUDA writeResource
+  status = cuGraphicsUnregisterResource(writeResource);
+  if (status != CUDA_SUCCESS) {
+    printf("Failed to unregister %d", status);
+    cleanup(FAILURE);
+  } else {
+    printf("Unregistered writeResource. \n");
+  }
+
+  // Clean up CUDA readResource
+  status = cuGraphicsUnregisterResource(readResource);
+  if (status != CUDA_SUCCESS) {
+    printf("Failed to unregister %d", status);
+    cleanup(FAILURE);
+  } else {
+    printf("Unregistered readResource. \n");
+  }
+}
+
+// Verifies the values in readSurface whether they are expected ones
+// And writes value of newData into writeSurface
+__global__ void verify_and_update_kernel(CUsurfObject write, CUsurfObject read,
+                                         char expected, char newval, int width,
+                                         int height) {
+  unsigned int x = blockDim.x * blockIdx.x + threadIdx.x;
+  unsigned int y = blockDim.y * blockIdx.y + threadIdx.y;
+
+  if (x < width && y < height) {
+    uchar4 check;
+    surf2Dread(&check, read, x * 4, y);
+    if (check.x != expected || check.y != expected || check.z != expected ||
+        check.w != expected) {
+      printf(
+          "Mismatch found in values read[0]= %u read[1]= %u read[2]= %u "
+          "read[3]= %u expected is %u\n",
+          check.x, check.y, check.z, check.w, expected);
+      numErrors++;
+      return;
+    }
+    uchar4 data = make_uchar4(newval, newval, newval, newval);
+    surf2Dwrite(data, write, x * 4, y);
+  }
+}
+
+__global__ void getNumErrors(int *numErr) { *numErr = numErrors; }
+
+extern "C" cudaError_t cudaGetValueMismatch() {
+  int numErr_h;
+  int *numErr_d = NULL;
+  cudaError_t err = cudaSuccess;
+
+  err = cudaMalloc(&numErr_d, sizeof(int));
+  if (err != cudaSuccess) {
+    printf("Cuda Main: cudaMemcpy failed with %s\n", cudaGetErrorString(err));
+    cudaFree(numErr_d);
+    return err;
+  }
+
+  getNumErrors<<<1, 1>>>(numErr_d);
+  err = cudaDeviceSynchronize();
+  if (err != cudaSuccess) {
+    printf("Cuda Main: cudaDeviceSynchronize failed with %s\n",
+           cudaGetErrorString(err));
+  }
+  err = cudaMemcpy(&numErr_h, numErr_d, sizeof(int), cudaMemcpyDeviceToHost);
+  if (err != cudaSuccess) {
+    printf("Cuda Main: cudaMemcpy failed with %s\n", cudaGetErrorString(err));
+    cudaFree(numErr_d);
+    return err;
+  }
+  err = cudaFree(numErr_d);
+  if (err != cudaSuccess) {
+    printf("Cuda Main: cudaFree failed with %s\n", cudaGetErrorString(err));
+    return err;
+  }
+  if (numErr_h > 0) {
+    return cudaErrorUnknown;
+  }
+  return cudaSuccess;
+}
+
+// Clean up state and exit. If status is SUCCESS, regression success is printed
+// to stdout. This will happen if the glut timer is triggered. If status is
+// anything else, the regression failure message is printed.
+void cleanup(int status) {
+  GLenum glErr = GL_NO_ERROR;
+  cudaError cudaErr = cudaSuccess;
+  int exitStatus = status;
+
+  // Clean up GL
+  if (*tex) {
+    GL_SAFE_CALL_NO_CLEANUP(glDeleteTextures(2, tex), glErr);
+  }
+
+  // Print test status and exit
+  if (glErr != GL_NO_ERROR || cudaErr != cudaSuccess) exitStatus = FAILURE;
+
+  printStatus(exitStatus);
+
+  cleanExit = 1;
+
+  graphics_close_window();
+
+  if (exitStatus == FAILURE) exit(EXIT_FAILURE);
+
+  if (exitStatus == WAIVED) exit(EXIT_WAIVED);
+
+  exit(0);
+}
+
+void exitHandler(void) {
+  if (!cleanExit) {
+    printf("&&&& EGLSync_CUDAEvent_Interop unexpected failure \n");
+    printStatus(FAILURE);
+  }
+}
+
+// Print test success or fail for regression testing
+void printStatus(int status) {
+  switch (status) {
+    case SUCCESS:
+      printf("&&&& EGLSync_CUDAEvent_Interop PASSED\n");
+      break;
+    case WAIVED:
+      printf("&&&& EGLSync_CUDAEvent_Interop WAIVED\n");
+      break;
+    default:
+      printf("&&&& EGLSync_CUDAEvent_Interop FAILED\n");
+      break;
+  }
+  fflush(stdout);
+}

Samples/EGLSync_CUDAEvent_Interop/Makefile

@@ -0,0 +1,457 @@
+################################################################################
+# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#  * Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#  * Neither the name of NVIDIA CORPORATION nor the names of its
+#    contributors may be used to endorse or promote products derived
+#    from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+################################################################################
+#
+# Makefile project only supported on Mac OS X and Linux Platforms)
+#
+################################################################################
+
+# Location of the CUDA Toolkit
+CUDA_PATH ?= /usr/local/cuda
+
+##############################
+# start deprecated interface #
+##############################
+ifeq ($(x86_64),1)
+    $(info WARNING - x86_64 variable has been deprecated)
+    $(info WARNING - please use TARGET_ARCH=x86_64 instead)
+    TARGET_ARCH ?= x86_64
+endif
+ifeq ($(ARMv7),1)
+    $(info WARNING - ARMv7 variable has been deprecated)
+    $(info WARNING - please use TARGET_ARCH=armv7l instead)
+    TARGET_ARCH ?= armv7l
+endif
+ifeq ($(aarch64),1)
+    $(info WARNING - aarch64 variable has been deprecated)
+    $(info WARNING - please use TARGET_ARCH=aarch64 instead)
+    TARGET_ARCH ?= aarch64
+endif
+ifeq ($(ppc64le),1)
+    $(info WARNING - ppc64le variable has been deprecated)
+    $(info WARNING - please use TARGET_ARCH=ppc64le instead)
+    TARGET_ARCH ?= ppc64le
+endif
+ifneq ($(GCC),)
+    $(info WARNING - GCC variable has been deprecated)
+    $(info WARNING - please use HOST_COMPILER=$(GCC) instead)
+    HOST_COMPILER ?= $(GCC)
+endif
+ifneq ($(abi),)
+    $(error ERROR - abi variable has been removed)
+endif
+############################
+# end deprecated interface #
+############################
+
+# architecture
+HOST_ARCH   := $(shell uname -m)
+TARGET_ARCH ?= $(HOST_ARCH)
+ifneq (,$(filter $(TARGET_ARCH),x86_64 aarch64 sbsa ppc64le armv7l))
+    ifneq ($(TARGET_ARCH),$(HOST_ARCH))
+        ifneq (,$(filter $(TARGET_ARCH),x86_64 aarch64 sbsa ppc64le))
+            TARGET_SIZE := 64
+        else ifneq (,$(filter $(TARGET_ARCH),armv7l))
+            TARGET_SIZE := 32
+        endif
+    else
+        TARGET_SIZE := $(shell getconf LONG_BIT)
+    endif
+else
+    $(error ERROR - unsupported value $(TARGET_ARCH) for TARGET_ARCH!)
+endif
+
+# sbsa and aarch64 systems look similar. Need to differentiate them at host level for now.
+ifeq ($(HOST_ARCH),aarch64)
+    ifeq ($(CUDA_PATH)/targets/sbsa-linux,$(shell ls -1d $(CUDA_PATH)/targets/sbsa-linux 2>/dev/null))
+        HOST_ARCH := sbsa
+        TARGET_ARCH := sbsa
+    endif
+endif
+
+ifneq ($(TARGET_ARCH),$(HOST_ARCH))
+    ifeq (,$(filter $(HOST_ARCH)-$(TARGET_ARCH),aarch64-armv7l x86_64-armv7l x86_64-aarch64 x86_64-sbsa x86_64-ppc64le))
+        $(error ERROR - cross compiling from $(HOST_ARCH) to $(TARGET_ARCH) is not supported!)
+    endif
+endif
+
+# When on native aarch64 system with userspace of 32-bit, change TARGET_ARCH to armv7l
+ifeq ($(HOST_ARCH)-$(TARGET_ARCH)-$(TARGET_SIZE),aarch64-aarch64-32)
+    TARGET_ARCH = armv7l
+endif
+
+# operating system
+HOST_OS   := $(shell uname -s 2>/dev/null | tr "[:upper:]" "[:lower:]")
+TARGET_OS ?= $(HOST_OS)
+ifeq (,$(filter $(TARGET_OS),linux darwin qnx android))
+    $(error ERROR - unsupported value $(TARGET_OS) for TARGET_OS!)
+endif
+
+# host compiler
+ifeq ($(TARGET_OS),darwin)
+    ifeq ($(shell expr `xcodebuild -version | grep -i xcode | awk '{print $$2}' | cut -d'.' -f1` \>= 5),1)
+        HOST_COMPILER ?= clang++
+    endif
+else ifneq ($(TARGET_ARCH),$(HOST_ARCH))
+    ifeq ($(HOST_ARCH)-$(TARGET_ARCH),x86_64-armv7l)
+        ifeq ($(TARGET_OS),linux)
+            HOST_COMPILER ?= arm-linux-gnueabihf-g++
+        else ifeq ($(TARGET_OS),qnx)
+            ifeq ($(QNX_HOST),)
+                $(error ERROR - QNX_HOST must be passed to the QNX host toolchain)
+            endif
+            ifeq ($(QNX_TARGET),)
+                $(error ERROR - QNX_TARGET must be passed to the QNX target toolchain)
+            endif
+            export QNX_HOST
+            export QNX_TARGET
+            HOST_COMPILER ?= $(QNX_HOST)/usr/bin/arm-unknown-nto-qnx6.6.0eabi-g++
+        else ifeq ($(TARGET_OS),android)
+            HOST_COMPILER ?= arm-linux-androideabi-g++
+        endif
+    else ifeq ($(TARGET_ARCH),aarch64)
+        ifeq ($(TARGET_OS), linux)
+            HOST_COMPILER ?= aarch64-linux-gnu-g++
+        else ifeq ($(TARGET_OS),qnx)
+            ifeq ($(QNX_HOST),)
+                $(error ERROR - QNX_HOST must be passed to the QNX host toolchain)
+            endif
+            ifeq ($(QNX_TARGET),)
+                $(error ERROR - QNX_TARGET must be passed to the QNX target toolchain)
+            endif
+            export QNX_HOST
+            export QNX_TARGET
+            HOST_COMPILER ?= $(QNX_HOST)/usr/bin/q++
+        else ifeq ($(TARGET_OS), android)
+            HOST_COMPILER ?= aarch64-linux-android-clang++
+        endif
+    else ifeq ($(TARGET_ARCH),sbsa)
+        HOST_COMPILER ?= aarch64-linux-gnu-g++
+    else ifeq ($(TARGET_ARCH),ppc64le)
+        HOST_COMPILER ?= powerpc64le-linux-gnu-g++
+    endif
+endif
+HOST_COMPILER ?= g++
+NVCC          := $(CUDA_PATH)/bin/nvcc -ccbin $(HOST_COMPILER)
+
+# internal flags
+NVCCFLAGS   := -m${TARGET_SIZE}
+CCFLAGS     :=
+LDFLAGS     :=
+
+# build flags
+ifeq ($(TARGET_OS),darwin)
+    LDFLAGS += -rpath $(CUDA_PATH)/lib
+    CCFLAGS += -arch $(HOST_ARCH)
+else ifeq ($(HOST_ARCH)-$(TARGET_ARCH)-$(TARGET_OS),x86_64-armv7l-linux)
+    LDFLAGS += --dynamic-linker=/lib/ld-linux-armhf.so.3
+    CCFLAGS += -mfloat-abi=hard
+else ifeq ($(TARGET_OS),android)
+    LDFLAGS += -pie
+    CCFLAGS += -fpie -fpic -fexceptions
+endif
+
+ifneq ($(TARGET_ARCH),$(HOST_ARCH))
+    ifeq ($(TARGET_ARCH)-$(TARGET_OS),armv7l-linux)
+        ifneq ($(TARGET_FS),)
+            GCCVERSIONLTEQ46 := $(shell expr `$(HOST_COMPILER) -dumpversion` \<= 4.6)
+            ifeq ($(GCCVERSIONLTEQ46),1)
+                CCFLAGS += --sysroot=$(TARGET_FS)
+            endif
+            LDFLAGS += --sysroot=$(TARGET_FS)
+            LDFLAGS += -rpath-link=$(TARGET_FS)/lib
+            LDFLAGS += -rpath-link=$(TARGET_FS)/usr/lib
+            LDFLAGS += -rpath-link=$(TARGET_FS)/usr/lib/arm-linux-gnueabihf
+        endif
+    endif
+    ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-linux)
+        ifneq ($(TARGET_FS),)
+            GCCVERSIONLTEQ46 := $(shell expr `$(HOST_COMPILER) -dumpversion` \<= 4.6)
+            ifeq ($(GCCVERSIONLTEQ46),1)
+                CCFLAGS += --sysroot=$(TARGET_FS)
+            endif
+            LDFLAGS += --sysroot=$(TARGET_FS)
+            LDFLAGS += -rpath-link=$(TARGET_FS)/lib -L$(TARGET_FS)/lib
+            LDFLAGS += -rpath-link=$(TARGET_FS)/lib/aarch64-linux-gnu -L$(TARGET_FS)/lib/aarch64-linux-gnu
+            LDFLAGS += -rpath-link=$(TARGET_FS)/usr/lib -L$(TARGET_FS)/usr/lib
+            LDFLAGS += -rpath-link=$(TARGET_FS)/usr/lib/aarch64-linux-gnu -L$(TARGET_FS)/usr/lib/aarch64-linux-gnu
+            LDFLAGS += --unresolved-symbols=ignore-in-shared-libs
+            CCFLAGS += -isystem=$(TARGET_FS)/usr/include -I$(TARGET_FS)/usr/include -I$(TARGET_FS)/usr/include/libdrm
+            CCFLAGS += -isystem=$(TARGET_FS)/usr/include/aarch64-linux-gnu -I$(TARGET_FS)/usr/include/aarch64-linux-gnu
+        endif
+    endif
+    ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-qnx)
+        NVCCFLAGS += --qpp-config 5.4.0,gcc_ntoaarch64le
+        CCFLAGS += -DWIN_INTERFACE_CUSTOM -I/usr/include/aarch64-qnx-gnu
+        LDFLAGS += -lsocket
+        LDFLAGS += -L/usr/lib/aarch64-qnx-gnu
+        CCFLAGS += "-Wl\,-rpath-link\,/usr/lib/aarch64-qnx-gnu"
+        ifdef TARGET_OVERRIDE
+            LDFLAGS += -lslog2
+        endif
+
+        ifneq ($(TARGET_FS),)
+            LDFLAGS += -L$(TARGET_FS)/usr/lib
+            CCFLAGS += "-Wl\,-rpath-link\,$(TARGET_FS)/usr/lib"
+            LDFLAGS += -L$(TARGET_FS)/usr/libnvidia
+            CCFLAGS += "-Wl\,-rpath-link\,$(TARGET_FS)/usr/libnvidia"
+            CCFLAGS += -I$(TARGET_FS)/../include
+        endif
+    endif
+endif
+
+ifdef TARGET_OVERRIDE # cuda toolkit targets override
+    NVCCFLAGS += -target-dir $(TARGET_OVERRIDE)
+endif
+
+# Install directory of different arch
+CUDA_INSTALL_TARGET_DIR :=
+ifeq ($(TARGET_ARCH)-$(TARGET_OS),armv7l-linux)
+    CUDA_INSTALL_TARGET_DIR = targets/armv7-linux-gnueabihf/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-linux)
+    CUDA_INSTALL_TARGET_DIR = targets/aarch64-linux/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),sbsa-linux)
+    CUDA_INSTALL_TARGET_DIR = targets/sbsa-linux/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),armv7l-android)
+    CUDA_INSTALL_TARGET_DIR = targets/armv7-linux-androideabi/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-android)
+    CUDA_INSTALL_TARGET_DIR = targets/aarch64-linux-androideabi/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),armv7l-qnx)
+    CUDA_INSTALL_TARGET_DIR = targets/ARMv7-linux-QNX/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-qnx)
+    CUDA_INSTALL_TARGET_DIR = targets/aarch64-qnx/
+else ifeq ($(TARGET_ARCH),ppc64le)
+    CUDA_INSTALL_TARGET_DIR = targets/ppc64le-linux/
+endif
+
+# Debug build flags
+ifeq ($(dbg),1)
+      NVCCFLAGS += -g -G
+      BUILD_TYPE := debug
+else
+      BUILD_TYPE := release
+endif
+
+ALL_CCFLAGS :=
+ALL_CCFLAGS += $(NVCCFLAGS)
+ALL_CCFLAGS += $(EXTRA_NVCCFLAGS)
+ALL_CCFLAGS += $(addprefix -Xcompiler ,$(CCFLAGS))
+ALL_CCFLAGS += $(addprefix -Xcompiler ,$(EXTRA_CCFLAGS))
+
+UBUNTU = $(shell lsb_release -i -s 2>/dev/null | grep -i ubuntu)
+
+SAMPLE_ENABLED := 1
+
+# This sample is not supported on Linux x86_64
+ifeq ($(TARGET_OS),linux)
+  ifeq ($(TARGET_ARCH),x86_64)
+    $(info >>> WARNING - EGLSync_CUDAEvent_Interop is not supported on Linux x86_64 - waiving sample <<<)
+    SAMPLE_ENABLED := 0
+  endif
+endif
+
+# This sample is not supported on Mac OSX
+ifeq ($(TARGET_OS),darwin)
+  $(info >>> WARNING - EGLSync_CUDAEvent_Interop is not supported on Mac OSX - waiving sample <<<)
+  SAMPLE_ENABLED := 0
+endif
+
+# This sample is not supported on android
+ifeq ($(TARGET_OS),android)
+  $(info >>> WARNING - EGLSync_CUDAEvent_Interop is not supported on android - waiving sample <<<)
+  SAMPLE_ENABLED := 0
+endif
+
+# This sample is not supported on QNX
+ifeq ($(TARGET_OS),qnx)
+  $(info >>> WARNING - EGLSync_CUDAEvent_Interop is not supported on QNX - waiving sample <<<)
+  SAMPLE_ENABLED := 0
+endif
+
+ALL_LDFLAGS :=
+ALL_LDFLAGS += $(ALL_CCFLAGS)
+ALL_LDFLAGS += $(addprefix -Xlinker ,$(LDFLAGS))
+ALL_LDFLAGS += $(addprefix -Xlinker ,$(EXTRA_LDFLAGS))
+
+# Common includes and paths for CUDA
+INCLUDES  := -I../../Common
+LIBRARIES :=
+
+################################################################################
+
+# Makefile include to help find EGL Libraries
+include ./findegl.mk
+
+# EGL specific libraries
+ifneq ($(TARGET_OS),darwin)
+ LIBRARIES += -lEGL
+endif
+
+#Detect if installed version of GCC supports required C++11
+ifeq ($(TARGET_OS),linux)
+    empty :=
+    space := $(empty) $(empty)
+    GCCVERSIONSTRING := $(shell expr `$(HOST_COMPILER) -dumpversion`)
+#Create version number without "."
+    GCCVERSION := $(shell expr `echo $(GCCVERSIONSTRING)` | cut -f1 -d.)
+    GCCVERSION += $(shell expr `echo $(GCCVERSIONSTRING)` | cut -f2 -d.)
+    GCCVERSION += $(shell expr `echo $(GCCVERSIONSTRING)` | cut -f3 -d.)
+# Make sure the version number has at least 3 decimals
+    GCCVERSION += 00
+# Remove spaces from the version number
+    GCCVERSION := $(subst $(space),$(empty),$(GCCVERSION))
+#$(warning $(GCCVERSION))
+
+    IS_MIN_VERSION := $(shell expr `echo $(GCCVERSION)` \>= 51000)
+
+    ifeq ($(IS_MIN_VERSION), 1)
+        $(info >>> GCC Version is greater or equal to 5.1.0 <<<)
+    else
+        $(info >>> Waiving build. Minimum GCC version required is 5.1.0<<<)
+        SAMPLE_ENABLED := 0
+    endif
+endif
+
+# Gencode arguments
+ifeq ($(TARGET_ARCH),$(filter $(TARGET_ARCH),armv7l aarch64))
+SMS ?= 35 37 50 52 60 61 70 72 75 80 86
+else
+SMS ?= 35 37 50 52 60 61 70 75 80 86
+endif
+
+ifeq ($(SMS),)
+$(info >>> WARNING - no SM architectures have been specified - waiving sample <<<)
+SAMPLE_ENABLED := 0
+endif
+
+ifeq ($(GENCODE_FLAGS),)
+# Generate SASS code for each SM architecture listed in $(SMS)
+$(foreach sm,$(SMS),$(eval GENCODE_FLAGS += -gencode arch=compute_$(sm),code=sm_$(sm)))
+
+# Generate PTX code from the highest SM architecture in $(SMS) to guarantee forward-compatibility
+HIGHEST_SM := $(lastword $(sort $(SMS)))
+ifneq ($(HIGHEST_SM),)
+GENCODE_FLAGS += -gencode arch=compute_$(HIGHEST_SM),code=compute_$(HIGHEST_SM)
+endif
+endif
+
+ifeq ($(TARGET_OS),darwin)
+  ALL_LDFLAGS += -Xcompiler -F/Library/Frameworks -Xlinker -framework -Xlinker CUDA
+else
+  ifeq ($(TARGET_ARCH),x86_64)
+    CUDA_SEARCH_PATH ?= $(CUDA_PATH)/lib64/stubs
+    CUDA_SEARCH_PATH += $(CUDA_PATH)/targets/x86_64-linux/lib/stubs
+  endif
+
+  ifeq ($(TARGET_ARCH)-$(TARGET_OS),armv7l-linux)
+    CUDA_SEARCH_PATH ?= $(CUDA_PATH)/targets/armv7-linux-gnueabihf/lib/stubs
+  endif
+
+  ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-linux)
+    CUDA_SEARCH_PATH ?= $(CUDA_PATH)/targets/aarch64-linux/lib/stubs
+  endif
+
+  ifeq ($(TARGET_ARCH)-$(TARGET_OS),sbsa-linux)
+    CUDA_SEARCH_PATH ?= $(CUDA_PATH)/targets/sbsa-linux/lib/stubs
+  endif
+
+  ifeq ($(TARGET_ARCH)-$(TARGET_OS),armv7l-android)
+    CUDA_SEARCH_PATH ?= $(CUDA_PATH)/targets/armv7-linux-androideabi/lib/stubs
+  endif
+
+  ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-android)
+    CUDA_SEARCH_PATH ?= $(CUDA_PATH)/targets/aarch64-linux-androideabi/lib/stubs
+  endif
+
+  ifeq ($(TARGET_ARCH)-$(TARGET_OS),armv7l-qnx)
+    CUDA_SEARCH_PATH ?= $(CUDA_PATH)/targets/ARMv7-linux-QNX/lib/stubs
+  endif
+
+  ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-qnx)
+    CUDA_SEARCH_PATH ?= $(CUDA_PATH)/targets/aarch64-qnx/lib/stubs
+    ifdef TARGET_OVERRIDE
+        CUDA_SEARCH_PATH := $(CUDA_PATH)/targets/$(TARGET_OVERRIDE)/lib/stubs
+    endif
+  endif
+
+  ifeq ($(TARGET_ARCH),ppc64le)
+    CUDA_SEARCH_PATH ?= $(CUDA_PATH)/targets/ppc64le-linux/lib/stubs
+  endif
+
+  ifeq ($(HOST_ARCH),ppc64le)
+    CUDA_SEARCH_PATH += $(CUDA_PATH)/lib64/stubs
+  endif
+
+  CUDALIB ?= $(shell find -L $(CUDA_SEARCH_PATH) -maxdepth 1 -name libcuda.so 2> /dev/null)
+  ifeq ("$(CUDALIB)","")
+    $(info >>> WARNING - libcuda.so not found, CUDA Driver is not installed.  Please re-install the driver. <<<)
+    SAMPLE_ENABLED := 0
+  else
+    CUDALIB := $(shell echo $(CUDALIB) | sed "s/ .*//" | sed "s/\/libcuda.so//" )
+    LIBRARIES += -L$(CUDALIB) -lcuda
+  endif
+endif
+
+ALL_CCFLAGS += --threads 0 --std=c++11
+
+LIBRARIES += -lX11 -lGLESv2
+
+ifeq ($(SAMPLE_ENABLED),0)
+EXEC ?= @echo "[@]"
+endif
+
+################################################################################
+
+# Target rules
+all: build
+
+build: EGLSync_CUDAEvent_Interop
+
+check.deps:
+ifeq ($(SAMPLE_ENABLED),0)
+	@echo "Sample will be waived due to the above missing dependencies"
+else
+	@echo "Sample is ready - all dependencies have been met"
+endif
+
+EGLSync_CUDAEvent_Interop.o:EGLSync_CUDAEvent_Interop.cu
+	$(EXEC) $(NVCC) $(INCLUDES) $(ALL_CCFLAGS) $(GENCODE_FLAGS) -o $@ -c $<
+
+EGLSync_CUDAEvent_Interop: EGLSync_CUDAEvent_Interop.o
+	$(EXEC) $(NVCC) $(ALL_LDFLAGS) $(GENCODE_FLAGS) -o $@ $+ $(LIBRARIES)
+	$(EXEC) mkdir -p ../../bin/$(TARGET_ARCH)/$(TARGET_OS)/$(BUILD_TYPE)
+	$(EXEC) cp $@ ../../bin/$(TARGET_ARCH)/$(TARGET_OS)/$(BUILD_TYPE)
+
+run: build
+	$(EXEC) ./EGLSync_CUDAEvent_Interop
+
+clean:
+	rm -f EGLSync_CUDAEvent_Interop EGLSync_CUDAEvent_Interop.o
+	rm -rf ../../bin/$(TARGET_ARCH)/$(TARGET_OS)/$(BUILD_TYPE)/EGLSync_CUDAEvent_Interop
+
+clobber: clean

Samples/EGLSync_CUDAEvent_Interop/NsightEclipse.xml

@@ -0,0 +1,75 @@
+<?xml version="1.0" encoding="UTF-8"?> 
+<!DOCTYPE entry SYSTEM "SamplesInfo.dtd">
+<entry>
+  <name>EGLSync_CUDAEvent_Interop</name>
+  <cuda_api_list>
+    <driver>cuDeviceGet</driver>
+    <driver>cuDeviceGetAttribute</driver>
+    <driver>cuDeviceComputeCapability</driver>
+    <driver>cuDeviceGetCount</driver>
+    <driver>cuDeviceGetName</driver>
+    <driver>cuCtxCreate</driver>
+    <driver>cuMemAlloc</driver>
+    <driver>cuMemFree</driver>
+    <driver>cuMemcpy3D</driver>
+    <driver>cuStreamCreate</driver>
+    <driver>cuCtxPushCurrent</driver>
+    <driver>cuCtxPopCurrent</driver>
+    <driver>cuCtxDestroy</driver>
+  </cuda_api_list>
+  <description><![CDATA[Demonstrates interoperability between CUDA Event and EGL Sync/EGL Image using which one can achieve synchronization on GPU itself for GL-EGL-CUDA operations instead of blocking CPU for synchronization.]]></description>
+  <devicecompilation>whole</devicecompilation>
+  <includepaths>
+    <path>./</path>
+    <path>../</path>
+    <path>../../Common</path>
+  </includepaths>
+  <keyconcepts>
+    <concept level="basic">EGLSync-CUDAEvent Interop</concept>
+    <concept level="basic">EGLImage-CUDA Interop</concept>
+  </keyconcepts>
+  <keywords>
+    <keyword>EGL Sync</keyword>
+  </keywords>
+  <libraries>
+    <library os="linux">cuda</library>
+    <library framework="true" os="macosx">CUDA</library>
+    <library>X11</library>
+    <library>GLESv2</library>
+  </libraries>
+  <librarypaths>
+  </librarypaths>
+  <nsight_eclipse>true</nsight_eclipse>
+  <primary_file>EGLSync_CUDAEvent_Interop.cu</primary_file>
+  <required_dependencies>
+    <dependency>EGL</dependency>
+    <dependency>EGLSync</dependency>
+    <dependency>X11</dependency>
+    <dependency>GLES</dependency>
+  </required_dependencies>
+  <scopes>
+    <scope>1:CUDA Basic Topics</scope>
+    <scope>2:Graphics Interop</scope>
+  </scopes>
+  <sm-arch>sm35</sm-arch>
+  <sm-arch>sm37</sm-arch>
+  <sm-arch>sm50</sm-arch>
+  <sm-arch>sm52</sm-arch>
+  <sm-arch>sm60</sm-arch>
+  <sm-arch>sm61</sm-arch>
+  <sm-arch>sm70</sm-arch>
+  <sm-arch>sm72</sm-arch>
+  <sm-arch>sm75</sm-arch>
+  <sm-arch>sm80</sm-arch>
+  <sm-arch>sm86</sm-arch>
+  <supported_envs>
+    <env>
+      <arch>arm</arch>
+    </env>
+  </supported_envs>
+  <supported_sm_architectures>
+    <include>all</include>
+  </supported_sm_architectures>
+  <title>EGLSync CUDA Event Interop</title>
+  <type>exe</type>
+</entry>

Samples/EGLSync_CUDAEvent_Interop/README.md

@@ -0,0 +1,37 @@
+# EGLSync_CUDAEvent_Interop - EGLSync CUDA Event Interop
+
+## Description
+
+Demonstrates interoperability between CUDA Event and EGL Sync/EGL Image using which one can achieve synchronization on GPU itself for GL-EGL-CUDA operations instead of blocking CPU for synchronization.
+
+## Key Concepts
+
+EGLSync-CUDAEvent Interop, EGLImage-CUDA Interop
+
+## Supported SM Architectures
+
+[SM 3.5 ](https://developer.nvidia.com/cuda-gpus)  [SM 3.7 ](https://developer.nvidia.com/cuda-gpus)  [SM 5.0 ](https://developer.nvidia.com/cuda-gpus)  [SM 5.2 ](https://developer.nvidia.com/cuda-gpus)  [SM 6.0 ](https://developer.nvidia.com/cuda-gpus)  [SM 6.1 ](https://developer.nvidia.com/cuda-gpus)  [SM 7.0 ](https://developer.nvidia.com/cuda-gpus)  [SM 7.2 ](https://developer.nvidia.com/cuda-gpus)  [SM 7.5 ](https://developer.nvidia.com/cuda-gpus)  [SM 8.0 ](https://developer.nvidia.com/cuda-gpus)  [SM 8.6 ](https://developer.nvidia.com/cuda-gpus)
+
+## Supported OSes
+
+## Supported CPU Architecture
+
+armv7l
+
+## CUDA APIs involved
+
+### [CUDA Driver API](http://docs.nvidia.com/cuda/cuda-driver-api/index.html)
+cuDeviceGet, cuDeviceGetAttribute, cuDeviceComputeCapability, cuDeviceGetCount, cuDeviceGetName, cuCtxCreate, cuMemAlloc, cuMemFree, cuMemcpy3D, cuStreamCreate, cuCtxPushCurrent, cuCtxPopCurrent, cuCtxDestroy
+
+## Dependencies needed to build/run
+[EGL](../../README.md#egl), [EGLSync](../../README.md#eglsync), [X11](../../README.md#x11), [GLES](../../README.md#gles)
+
+## Prerequisites
+
+Download and install the [CUDA Toolkit 11.5](https://developer.nvidia.com/cuda-downloads) for your corresponding platform.
+Make sure the dependencies mentioned in [Dependencies]() section above are installed.
+
+## Build and Run
+
+## References (for more details)
+

Samples/EGLSync_CUDAEvent_Interop/egl_common.h

@@ -0,0 +1,71 @@
+/* Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+//
+// DESCRIPTION:   Common EGL functions header file
+//
+
+#ifndef _EGL_COMMON_H_
+#define _EGL_COMMON_H_
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <sys/stat.h>
+#include <sys/time.h>
+#include <signal.h>
+#include "cuda.h"
+#include "cudaEGL.h"
+
+EGLImageKHR eglImage;
+
+#define EXTENSION_LIST(T)                                \
+    T(PFNEGLCREATEIMAGEKHRPROC, eglCreateImageKHR)       \
+    T(PFNEGLDESTROYIMAGEKHRPROC, eglDestroyImageKHR)     \
+    T(PFNEGLCREATESYNCKHRPROC, eglCreateSyncKHR)         \
+    T(PFNEGLDESTROYSYNCKHRPROC, eglDestroySyncKHR)       \
+    T(PFNEGLCLIENTWAITSYNCKHRPROC, eglClientWaitSyncKHR) \
+    T(PFNEGLGETSYNCATTRIBKHRPROC, eglGetSyncAttribKHR)   \
+    T(PFNEGLCREATESYNC64KHRPROC, eglCreateSync64KHR)     \
+    T(PFNEGLWAITSYNCKHRPROC, eglWaitSyncKHR)
+
+#define eglCreateImageKHR my_eglCreateImageKHR
+#define eglDestroyImageKHR my_eglDestroyImageKHR
+#define eglCreateSyncKHR my_eglCreateSyncKHR
+#define eglDestroySyncKHR my_eglDestroySyncKHR
+#define eglClientWaitSyncKHR my_eglClientWaitSyncKHR
+#define eglGetSyncAttribKHR my_eglGetSyncAttribKHR
+#define eglCreateSync64KHR my_eglCreateSync64KHR
+#define eglWaitSyncKHR my_eglWaitSyncKHR
+
+#define EXTLST_DECL(tx, x) tx my_##x = NULL;
+#define EXTLST_EXTERN(tx, x) extern tx my_##x;
+#define EXTLST_ENTRY(tx, x) {(extlst_fnptr_t *)&my_##x, #x},
+
+int eglSetupExtensions(void);
+#endif

Samples/EGLSync_CUDAEvent_Interop/findegl.mk

@@ -0,0 +1,160 @@
+################################################################################
+# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#  * Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#  * Neither the name of NVIDIA CORPORATION nor the names of its
+#    contributors may be used to endorse or promote products derived
+#    from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+################################################################################
+#
+#  findegl.mk is used to find the necessary EGL Libraries for specific distributions
+#               this is supported on Linux
+#
+################################################################################
+
+# Determine OS platform and unix distribution
+ifeq ("$(TARGET_OS)","linux")
+   # first search lsb_release
+   DISTRO  = $(shell lsb_release -i -s 2>/dev/null | tr "[:upper:]" "[:lower:]")
+   ifeq ("$(DISTRO)","")
+     # second search and parse /etc/issue
+     DISTRO = $(shell more /etc/issue | awk '{print $$1}' | sed '1!d' | sed -e "/^$$/d" 2>/dev/null | tr "[:upper:]" "[:lower:]")
+     # ensure data from /etc/issue is valid
+     ifeq (,$(filter $(DISTRO),ubuntu fedora red rhel centos suse))
+       DISTRO = 
+     endif
+     ifeq ("$(DISTRO)","")
+       # third, we can search in /etc/os-release or /etc/{distro}-release
+       DISTRO = $(shell awk '/ID/' /etc/*-release | sed 's/ID=//' | grep -v "VERSION" | grep -v "ID" | grep -v "DISTRIB")
+     endif
+   endif
+endif
+
+ifeq ("$(TARGET_OS)","linux")
+    # $(info) >> findegl.mk -> LINUX path <<<)
+    # Each set of Linux Distros have different paths for where to find their OpenGL libraries reside
+    UBUNTU = $(shell echo $(DISTRO) | grep -i ubuntu      >/dev/null 2>&1; echo $$?)
+    FEDORA = $(shell echo $(DISTRO) | grep -i fedora      >/dev/null 2>&1; echo $$?)
+    RHEL   = $(shell echo $(DISTRO) | grep -i 'red\|rhel' >/dev/null 2>&1; echo $$?)
+    CENTOS = $(shell echo $(DISTRO) | grep -i centos      >/dev/null 2>&1; echo $$?)
+    SUSE   = $(shell echo $(DISTRO) | grep -i 'suse\|sles' >/dev/null 2>&1; echo $$?)
+    ifeq ("$(UBUNTU)","0")
+      ifeq ($(HOST_ARCH)-$(TARGET_ARCH),x86_64-armv7l)
+        GLPATH := /usr/arm-linux-gnueabihf/lib
+        GLLINK := -L/usr/arm-linux-gnueabihf/lib
+        ifneq ($(TARGET_FS),) 
+          GLPATH += $(TARGET_FS)/usr/lib/arm-linux-gnueabihf
+          GLLINK += -L$(TARGET_FS)/usr/lib/arm-linux-gnueabihf
+        endif
+      else ifeq ($(HOST_ARCH)-$(TARGET_ARCH),x86_64-aarch64)
+        GLPATH := /usr/aarch64-linux-gnu/lib
+        GLLINK := -L/usr/aarch64-linux-gnu/lib
+        ifneq ($(TARGET_FS),)
+          GLPATH += $(TARGET_FS)/usr/lib
+          GLPATH += $(TARGET_FS)/usr/lib/aarch64-linux-gnu
+          GLLINK += -L$(TARGET_FS)/usr/lib/aarch64-linux-gnu
+        endif 
+      else
+        UBUNTU_PKG_NAME = $(shell which dpkg >/dev/null 2>&1 && dpkg -l 'nvidia-*' | grep '^ii' | awk '{print $$2}' | head -1)
+        ifneq ("$(UBUNTU_PKG_NAME)","")
+          GLPATH    ?= /usr/lib/$(UBUNTU_PKG_NAME)
+          GLLINK    ?= -L/usr/lib/$(UBUNTU_PKG_NAME)
+        endif
+
+        DFLT_PATH ?= /usr/lib
+      endif
+    endif
+    ifeq ("$(SUSE)","0")
+      GLPATH    ?= /usr/X11R6/lib64
+      GLLINK    ?= -L/usr/X11R6/lib64
+      DFLT_PATH ?= /usr/lib64
+    endif
+    ifeq ("$(FEDORA)","0")
+      GLPATH    ?= /usr/lib64/nvidia
+      GLLINK    ?= -L/usr/lib64/nvidia
+      DFLT_PATH ?= /usr/lib64
+    endif
+    ifeq ("$(RHEL)","0")
+      GLPATH    ?= /usr/lib64/nvidia
+      GLLINK    ?= -L/usr/lib64/nvidia
+      DFLT_PATH ?= /usr/lib64
+    endif
+    ifeq ("$(CENTOS)","0")
+      GLPATH    ?= /usr/lib64/nvidia
+      GLLINK    ?= -L/usr/lib64/nvidia
+      DFLT_PATH ?= /usr/lib64
+    endif
+
+  EGLLIB  := $(shell find -L $(GLPATH) $(DFLT_PATH) -name libEGL.so    -print 2>/dev/null)
+
+  ifeq ("$(EGLLIB)","")
+      $(info >>> WARNING - libEGL.so not found, please install libEGL.so <<<)
+      SAMPLE_ENABLED := 0
+  endif
+
+  HEADER_SEARCH_PATH ?= $(TARGET_FS)/usr/include
+  ifeq ($(HOST_ARCH)-$(TARGET_ARCH)-$(TARGET_OS),x86_64-armv7l-linux)
+      HEADER_SEARCH_PATH += /usr/arm-linux-gnueabihf/include
+  else ifeq ($(HOST_ARCH)-$(TARGET_ARCH)-$(TARGET_OS),x86_64-aarch64-linux)
+      HEADER_SEARCH_PATH += /usr/aarch64-linux-gnu/include
+  endif
+
+  EGLHEADER  := $(shell find -L $(HEADER_SEARCH_PATH) -name egl.h -print 2>/dev/null)
+  EGLEXTHEADER  := $(shell find -L $(HEADER_SEARCH_PATH) -name eglext.h -print 2>/dev/null)
+
+  ifeq ("$(EGLHEADER)","")
+      $(info >>> WARNING - egl.h not found, please install egl.h <<<)
+      SAMPLE_ENABLED := 0
+  endif
+  ifeq ("$(EGLEXTHEADER)","")
+      $(info >>> WARNING - eglext.h not found, please install eglext.h <<<)
+      SAMPLE_ENABLED := 0
+  endif
+else
+endif
+
+ifeq ("$(TARGET_OS)","qnx")
+    HOST_CCFLAGS := -V5.4.0,gcc_ntoaarch64le
+endif
+
+# Attempt to compile a minimal EGL application and run to check if EGL_SUPPORT_REUSE_NV is supported in the EGL headers available.
+ifneq ($(SAMPLE_ENABLED), 0)
+      $(shell printf "#include <EGL/egl.h>\n#include <EGL/eglext.h>\nint main() {\n#ifdef EGL_SUPPORT_REUSE_NV \n #error \"Compatible EGL header found\" \n  return 0;\n#endif \n return 1;\n}"  > test.c; )
+      EGL_DEFINES := $(shell $(HOST_COMPILER) $(HOST_CCFLAGS) $(CCFLAGS) $(EXTRA_CCFLAGS) -lEGL test.c -c 2>&1 | grep -ic "Compatible EGL header found";)
+      SHOULD_WAIVE := 0
+      ifeq ($(EGL_DEFINES),0)
+        SHOULD_WAIVE := 1
+      endif
+      ifeq ($(SHOULD_WAIVE),1)
+          $(info -----------------------------------------------------------------------------------------------)
+          $(info WARNING - NVIDIA EGL EXTENSIONS are not available in the present EGL headers)
+          $(info -----------------------------------------------------------------------------------------------)
+          $(info   This CUDA Sample cannot be built if the EGL NVIDIA EXTENSIONS like EGL_SUPPORT_REUSE_NV are not supported in EGL headers.)
+          $(info   This will be a dry-run of the Makefile.)
+          $(info   Please install the latest khronos EGL headers and libs to build this sample)
+          $(info -----------------------------------------------------------------------------------------------)
+          SAMPLE_ENABLED := 0
+      endif
+      $(shell rm test.o test.c 2>/dev/null)
+endif
+

Samples/EGLSync_CUDAEvent_Interop/graphics_interface.h

@@ -0,0 +1,204 @@
+/* Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <X11/Xlib.h>
+#include <X11/Xutil.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdarg.h>
+
+Display *display;
+int screen;
+Window win = 0;
+
+void error_exit(const char *format, ...) {
+  va_list args;
+  va_start(args, format);
+  vfprintf(stderr, format, args);
+  va_end(args);
+  exit(1);
+}
+
+#include <EGL/egl.h>
+#include <EGL/eglext.h>
+
+#define GET_GLERROR(ret)                                                       \
+  {                                                                            \
+    GLenum err = glGetError();                                                 \
+    if (err != GL_NO_ERROR) {                                                  \
+      fprintf(stderr, "[%s line %d] OpenGL Error: 0x%x\n", __FILE__, __LINE__, \
+              err);                                                            \
+      fflush(stderr);                                                          \
+                                                                               \
+      switch (err) {                                                           \
+        case GL_INVALID_ENUM:                                                  \
+          printf("GL_INVALID_ENUM\n");                                         \
+          break;                                                               \
+        case GL_INVALID_VALUE:                                                 \
+          printf("GL_INVALID_VALUE\n");                                        \
+          break;                                                               \
+        case GL_INVALID_OPERATION:                                             \
+          printf("GL_INVALID_OPERATION\n");                                    \
+          break;                                                               \
+        case GL_OUT_OF_MEMORY:                                                 \
+          printf("GL_OUT_OF_MEMORY\n");                                        \
+          break;                                                               \
+        case GL_INVALID_FRAMEBUFFER_OPERATION:                                 \
+          printf("GL_INVALID_FRAMEBUFFER_OPERATION\n");                        \
+          break;                                                               \
+        default:                                                               \
+          printf("UKNOWN OPENGL ERROR CODE 0x%x\n", err);                      \
+      };                                                                       \
+    }                                                                          \
+  }
+
+EGLDisplay eglDisplay = EGL_NO_DISPLAY;
+EGLSurface eglSurface = EGL_NO_SURFACE;
+EGLContext eglContext = EGL_NO_CONTEXT;
+
+int graphics_setup_window(int xpos, int ypos, int width, int height,
+                          const char *windowname) {
+  EGLint configAttrs[] = {
+      EGL_RED_SIZE, 1, EGL_GREEN_SIZE, 1, EGL_BLUE_SIZE, 1, EGL_DEPTH_SIZE, 16,
+      EGL_SAMPLE_BUFFERS, 0, EGL_SAMPLES, 0,
+      // EGL_CONFORMANT,      EGL_OPENGL_BIT,
+      EGL_RENDERABLE_TYPE, EGL_OPENGL_ES2_BIT,  // 3_BIT_KHR,
+      EGL_NONE};
+  EGLint contextAttrs[] = {EGL_CONTEXT_CLIENT_VERSION, 3, EGL_NONE};
+
+  EGLConfig *configList = NULL;
+  EGLint configCount;
+
+  display = XOpenDisplay(NULL);
+  if (!display) error_exit("Error opening X display.\n");
+
+  screen = DefaultScreen(display);
+
+  eglDisplay = eglGetDisplay(0);
+
+  if (eglDisplay == EGL_NO_DISPLAY)
+    error_exit("EGL failed to obtain display\n");
+
+  if (!eglInitialize(eglDisplay, 0, 0))
+    error_exit("EGL failed to initialize\n");
+
+  if (!eglChooseConfig(eglDisplay, configAttrs, NULL, 0, &configCount) ||
+      !configCount)
+    error_exit("EGL failed to return any matching configurations\n");
+
+  configList = (EGLConfig *)malloc(configCount * sizeof(EGLConfig));
+
+  if (!eglChooseConfig(eglDisplay, configAttrs, configList, configCount,
+                       &configCount) ||
+      !configCount)
+    error_exit("EGL failed to populate configuration list\n");
+
+  Window xRootWindow = DefaultRootWindow(display);
+  XSetWindowAttributes xCreateWindowAttributes;
+  xCreateWindowAttributes.event_mask = ExposureMask;
+  win = XCreateWindow(display, xRootWindow, 0, 0, width, height, 0,
+                      CopyFromParent, InputOutput, CopyFromParent, CWEventMask,
+                      &xCreateWindowAttributes);
+  XMapWindow(display, win);
+
+  Atom netWmStateAtom = XInternAtom(display, "_NET_WM_STATE", false);
+  XEvent xEvent;
+  memset(&xEvent, 0, sizeof(xEvent));
+  xEvent.type = ClientMessage;
+  xEvent.xclient.window = win;
+  xEvent.xclient.message_type = netWmStateAtom;
+  xEvent.xclient.format = 32;
+  xEvent.xclient.data.l[0] = 1;
+  xEvent.xclient.data.l[1] = false;
+  XSendEvent(display, xRootWindow, false, SubstructureNotifyMask, &xEvent);
+
+  XStoreName(display, win, windowname);
+
+  XSelectInput(display, win,
+               ExposureMask | KeyPressMask | ButtonPressMask |
+                   ButtonReleaseMask | KeyReleaseMask | VisibilityChangeMask |
+                   PointerMotionMask);
+
+  EGLint windowAttrs[] = {EGL_NONE};
+
+  eglSurface = eglCreateWindowSurface(eglDisplay, configList[0],
+                                      (EGLNativeWindowType)win, windowAttrs);
+
+  if (!eglSurface) error_exit("EGL couldn't create window\n");
+
+  eglBindAPI(EGL_OPENGL_ES_API);
+
+  eglContext = eglCreateContext(eglDisplay, configList[0], NULL, contextAttrs);
+  if (!eglContext) error_exit("EGL couldn't create context\n");
+
+  if (!eglMakeCurrent(eglDisplay, eglSurface, eglSurface, eglContext))
+    error_exit("EGL couldn't make context/surface current\n");
+
+  EGLint Context_RendererType;
+  eglQueryContext(eglDisplay, eglContext, EGL_CONTEXT_CLIENT_TYPE,
+                  &Context_RendererType);
+
+  switch (Context_RendererType) {
+    case EGL_OPENGL_API:
+      printf("Using OpenGL API is not supported\n");
+      exit(EXIT_FAILURE);
+      break;
+    case EGL_OPENGL_ES_API:
+      printf("Using OpenGL ES API\n");
+      break;
+    case EGL_OPENVG_API:
+      error_exit("Context Query Returned OpenVG. This is Unsupported\n");
+    default:
+      error_exit("Unknown Context Type. %04X\n", Context_RendererType);
+  }
+
+  return 1;
+}
+
+void graphics_set_windowtitle(const char *windowname) {
+  XStoreName(display, win, windowname);
+}
+
+void graphics_swap_buffers() { eglSwapBuffers(eglDisplay, eglSurface); }
+
+void graphics_close_window() {
+  if (eglDisplay != EGL_NO_DISPLAY) {
+    eglMakeCurrent(eglDisplay, EGL_NO_SURFACE, EGL_NO_SURFACE, EGL_NO_CONTEXT);
+
+    if (eglContext != EGL_NO_CONTEXT) eglDestroyContext(eglDisplay, eglContext);
+
+    if (eglSurface != EGL_NO_SURFACE) eglDestroySurface(eglDisplay, eglSurface);
+
+    eglTerminate(eglDisplay);
+  }
+
+  if (display) {
+    if (win) XDestroyWindow(display, win);
+
+    XCloseDisplay(display);
+  }
+}

Samples/FDTD3d/.vscode/c_cpp_properties.json

@@ -0,0 +1,18 @@
+{
+    "configurations": [
+        {
+            "name": "Linux",
+            "includePath": [
+                "${workspaceFolder}/**",
+                "${workspaceFolder}/../../Common"
+            ],
+            "defines": [],
+            "compilerPath": "/usr/local/cuda/bin/nvcc",
+            "cStandard": "gnu17",
+            "cppStandard": "gnu++14",
+            "intelliSenseMode": "linux-gcc-x64",
+            "configurationProvider": "ms-vscode.makefile-tools"
+        }
+    ],
+    "version": 4
+}

Samples/FDTD3d/.vscode/extensions.json

@@ -0,0 +1,7 @@
+{
+    "recommendations": [
+        "nvidia.nsight-vscode-edition",
+        "ms-vscode.cpptools",
+        "ms-vscode.makefile-tools"
+    ]
+}

Samples/FDTD3d/.vscode/launch.json

@@ -0,0 +1,10 @@
+{
+    "configurations": [
+        {
+            "name": "CUDA C++: Launch",
+            "type": "cuda-gdb",
+            "request": "launch",
+            "program": "${workspaceFolder}/FDTD3d"
+        }
+    ]
+}

Samples/FDTD3d/.vscode/tasks.json

@@ -0,0 +1,15 @@
+{
+    "version": "2.0.0",
+    "tasks": [
+        {
+            "label": "sample",
+            "type": "shell",
+            "command": "make dbg=1",
+            "problemMatcher": ["$nvcc"],
+            "group": {
+                "kind": "build",
+                "isDefault": true
+            }
+        }
+    ]
+}

Samples/FDTD3d/FDTD3d_vs2017.sln

@@ -0,0 +1,20 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio 2017
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "FDTD3d", "FDTD3d_vs2017.vcxproj", "{997E0757-EA74-4A4E-A0FC-47D8C8831A15}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|x64 = Debug|x64
+		Release|x64 = Release|x64
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Debug|x64.ActiveCfg = Debug|x64
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Debug|x64.Build.0 = Debug|x64
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Release|x64.ActiveCfg = Release|x64
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Release|x64.Build.0 = Release|x64
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+EndGlobal

Samples/FDTD3d/FDTD3d_vs2017.vcxproj

@@ -0,0 +1,118 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <CUDAPropsPath Condition="'$(CUDAPropsPath)'==''">$(VCTargetsPath)\BuildCustomizations</CUDAPropsPath>
+  </PropertyGroup>
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Debug|x64">
+      <Configuration>Debug</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|x64">
+      <Configuration>Release</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>{997E0757-EA74-4A4E-A0FC-47D8C8831A15}</ProjectGuid>
+    <RootNamespace>FDTD3d_vs2017</RootNamespace>
+    <ProjectName>FDTD3d</ProjectName>
+    <CudaToolkitCustomDir />
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(WindowsTargetPlatformVersion)'==''">
+    <LatestTargetPlatformVersion>$([Microsoft.Build.Utilities.ToolLocationHelper]::GetLatestSDKTargetPlatformVersion('Windows', '10.0'))</LatestTargetPlatformVersion>
+    <WindowsTargetPlatformVersion Condition="'$(WindowsTargetPlatformVersion)' == ''">$(LatestTargetPlatformVersion)</WindowsTargetPlatformVersion>
+    <TargetPlatformVersion>$(WindowsTargetPlatformVersion)</TargetPlatformVersion>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <PropertyGroup>
+    <ConfigurationType>Application</ConfigurationType>
+    <CharacterSet>MultiByte</CharacterSet>
+    <PlatformToolset>v141</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)'=='Debug'">
+    <UseDebugLibraries>true</UseDebugLibraries>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)'=='Release'">
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <ImportGroup Label="ExtensionSettings">
+    <Import Project="$(CUDAPropsPath)\CUDA 11.5.props" />
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets">
+    <Import Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" />
+  </ImportGroup>
+  <PropertyGroup Label="UserMacros" />
+  <PropertyGroup>
+    <IntDir>$(Platform)/$(Configuration)/</IntDir>
+    <IncludePath>$(IncludePath)</IncludePath>
+    <CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRules />
+    <CodeAnalysisRuleAssemblies />
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Platform)'=='x64'">
+    <OutDir>../../bin/win64/$(Configuration)/</OutDir>
+  </PropertyGroup>
+  <ItemDefinitionGroup>
+    <ClCompile>
+      <WarningLevel>Level3</WarningLevel>
+      <PreprocessorDefinitions>WIN32;_MBCS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <AdditionalIncludeDirectories>./;$(CudaToolkitDir)/include;../../Common;inc;</AdditionalIncludeDirectories>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <AdditionalDependencies>cudart_static.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalLibraryDirectories>$(CudaToolkitLibDir);</AdditionalLibraryDirectories>
+      <OutputFile>$(OutDir)/FDTD3d.exe</OutputFile>
+    </Link>
+    <CudaCompile>
+      <CodeGeneration>compute_35,sm_35;compute_37,sm_37;compute_50,sm_50;compute_52,sm_52;compute_60,sm_60;compute_61,sm_61;compute_70,sm_70;compute_75,sm_75;compute_80,sm_80;compute_86,sm_86;</CodeGeneration>
+      <AdditionalOptions>-Xcompiler "/wd 4819" %(AdditionalOptions)</AdditionalOptions>
+      <Include>./;../../Common</Include>
+      <Defines>WIN32</Defines>
+      <AdditionalOptions>--threads 0</AdditionalOptions>
+    </CudaCompile>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)'=='Debug'">
+    <ClCompile>
+      <Optimization>Disabled</Optimization>
+      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <LinkTimeCodeGeneration>Default</LinkTimeCodeGeneration>
+    </Link>
+    <CudaCompile>
+      <Runtime>MTd</Runtime>
+      <TargetMachinePlatform>64</TargetMachinePlatform>
+    </CudaCompile>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)'=='Release'">
+    <ClCompile>
+      <Optimization>MaxSpeed</Optimization>
+      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <GenerateDebugInformation>false</GenerateDebugInformation>
+      <LinkTimeCodeGeneration>UseLinkTimeCodeGeneration</LinkTimeCodeGeneration>
+    </Link>
+    <CudaCompile>
+      <Runtime>MT</Runtime>
+      <TargetMachinePlatform>64</TargetMachinePlatform>
+    </CudaCompile>
+  </ItemDefinitionGroup>
+  <ItemGroup>
+    <ClCompile Include="src/FDTD3d.cpp" />
+    <CudaCompile Include="src/FDTD3dGPU.cu" />
+    <ClCompile Include="src/FDTD3dReference.cpp" />
+    <ClInclude Include="inc/FDTD3d.h" />
+    <ClInclude Include="inc/FDTD3dGPU.h" />
+    <ClInclude Include="inc/FDTD3dReference.h" />
+    <None Include="inc/FDTD3dGPUKernel.cuh" />
+  </ItemGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <ImportGroup Label="ExtensionTargets">
+    <Import Project="$(CUDAPropsPath)\CUDA 11.5.targets" />
+  </ImportGroup>
+</Project>

Samples/FDTD3d/FDTD3d_vs2019.sln

@@ -0,0 +1,20 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio 2019
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "FDTD3d", "FDTD3d_vs2019.vcxproj", "{997E0757-EA74-4A4E-A0FC-47D8C8831A15}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|x64 = Debug|x64
+		Release|x64 = Release|x64
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Debug|x64.ActiveCfg = Debug|x64
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Debug|x64.Build.0 = Debug|x64
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Release|x64.ActiveCfg = Release|x64
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Release|x64.Build.0 = Release|x64
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+EndGlobal

Samples/FDTD3d/FDTD3d_vs2019.vcxproj

@@ -0,0 +1,114 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <CUDAPropsPath Condition="'$(CUDAPropsPath)'==''">$(VCTargetsPath)\BuildCustomizations</CUDAPropsPath>
+  </PropertyGroup>
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Debug|x64">
+      <Configuration>Debug</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|x64">
+      <Configuration>Release</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>{997E0757-EA74-4A4E-A0FC-47D8C8831A15}</ProjectGuid>
+    <RootNamespace>FDTD3d_vs2019</RootNamespace>
+    <ProjectName>FDTD3d</ProjectName>
+    <CudaToolkitCustomDir />
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <PropertyGroup>
+    <ConfigurationType>Application</ConfigurationType>
+    <CharacterSet>MultiByte</CharacterSet>
+    <PlatformToolset>v142</PlatformToolset>
+    <WindowsTargetPlatformVersion>10.0</WindowsTargetPlatformVersion>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)'=='Debug'">
+    <UseDebugLibraries>true</UseDebugLibraries>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)'=='Release'">
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <ImportGroup Label="ExtensionSettings">
+    <Import Project="$(CUDAPropsPath)\CUDA 11.5.props" />
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets">
+    <Import Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" />
+  </ImportGroup>
+  <PropertyGroup Label="UserMacros" />
+  <PropertyGroup>
+    <IntDir>$(Platform)/$(Configuration)/</IntDir>
+    <IncludePath>$(IncludePath)</IncludePath>
+    <CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRules />
+    <CodeAnalysisRuleAssemblies />
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Platform)'=='x64'">
+    <OutDir>../../bin/win64/$(Configuration)/</OutDir>
+  </PropertyGroup>
+  <ItemDefinitionGroup>
+    <ClCompile>
+      <WarningLevel>Level3</WarningLevel>
+      <PreprocessorDefinitions>WIN32;_MBCS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <AdditionalIncludeDirectories>./;$(CudaToolkitDir)/include;../../Common;inc;</AdditionalIncludeDirectories>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <AdditionalDependencies>cudart_static.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalLibraryDirectories>$(CudaToolkitLibDir);</AdditionalLibraryDirectories>
+      <OutputFile>$(OutDir)/FDTD3d.exe</OutputFile>
+    </Link>
+    <CudaCompile>
+      <CodeGeneration>compute_35,sm_35;compute_37,sm_37;compute_50,sm_50;compute_52,sm_52;compute_60,sm_60;compute_61,sm_61;compute_70,sm_70;compute_75,sm_75;compute_80,sm_80;compute_86,sm_86;</CodeGeneration>
+      <AdditionalOptions>-Xcompiler "/wd 4819" %(AdditionalOptions)</AdditionalOptions>
+      <Include>./;../../Common</Include>
+      <Defines>WIN32</Defines>
+      <AdditionalOptions>--threads 0</AdditionalOptions>
+    </CudaCompile>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)'=='Debug'">
+    <ClCompile>
+      <Optimization>Disabled</Optimization>
+      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <LinkTimeCodeGeneration>Default</LinkTimeCodeGeneration>
+    </Link>
+    <CudaCompile>
+      <Runtime>MTd</Runtime>
+      <TargetMachinePlatform>64</TargetMachinePlatform>
+    </CudaCompile>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)'=='Release'">
+    <ClCompile>
+      <Optimization>MaxSpeed</Optimization>
+      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <GenerateDebugInformation>false</GenerateDebugInformation>
+      <LinkTimeCodeGeneration>UseLinkTimeCodeGeneration</LinkTimeCodeGeneration>
+    </Link>
+    <CudaCompile>
+      <Runtime>MT</Runtime>
+      <TargetMachinePlatform>64</TargetMachinePlatform>
+    </CudaCompile>
+  </ItemDefinitionGroup>
+  <ItemGroup>
+    <ClCompile Include="src/FDTD3d.cpp" />
+    <CudaCompile Include="src/FDTD3dGPU.cu" />
+    <ClCompile Include="src/FDTD3dReference.cpp" />
+    <ClInclude Include="inc/FDTD3d.h" />
+    <ClInclude Include="inc/FDTD3dGPU.h" />
+    <ClInclude Include="inc/FDTD3dReference.h" />
+    <None Include="inc/FDTD3dGPUKernel.cuh" />
+  </ItemGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <ImportGroup Label="ExtensionTargets">
+    <Import Project="$(CUDAPropsPath)\CUDA 11.5.targets" />
+  </ImportGroup>
+</Project>

Samples/FDTD3d/Makefile

@@ -0,0 +1,370 @@
+################################################################################
+# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#  * Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#  * Neither the name of NVIDIA CORPORATION nor the names of its
+#    contributors may be used to endorse or promote products derived
+#    from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+################################################################################
+#
+# Makefile project only supported on Mac OS X and Linux Platforms)
+#
+################################################################################
+
+# Location of the CUDA Toolkit
+CUDA_PATH ?= /usr/local/cuda
+
+##############################
+# start deprecated interface #
+##############################
+ifeq ($(x86_64),1)
+    $(info WARNING - x86_64 variable has been deprecated)
+    $(info WARNING - please use TARGET_ARCH=x86_64 instead)
+    TARGET_ARCH ?= x86_64
+endif
+ifeq ($(ARMv7),1)
+    $(info WARNING - ARMv7 variable has been deprecated)
+    $(info WARNING - please use TARGET_ARCH=armv7l instead)
+    TARGET_ARCH ?= armv7l
+endif
+ifeq ($(aarch64),1)
+    $(info WARNING - aarch64 variable has been deprecated)
+    $(info WARNING - please use TARGET_ARCH=aarch64 instead)
+    TARGET_ARCH ?= aarch64
+endif
+ifeq ($(ppc64le),1)
+    $(info WARNING - ppc64le variable has been deprecated)
+    $(info WARNING - please use TARGET_ARCH=ppc64le instead)
+    TARGET_ARCH ?= ppc64le
+endif
+ifneq ($(GCC),)
+    $(info WARNING - GCC variable has been deprecated)
+    $(info WARNING - please use HOST_COMPILER=$(GCC) instead)
+    HOST_COMPILER ?= $(GCC)
+endif
+ifneq ($(abi),)
+    $(error ERROR - abi variable has been removed)
+endif
+############################
+# end deprecated interface #
+############################
+
+# architecture
+HOST_ARCH   := $(shell uname -m)
+TARGET_ARCH ?= $(HOST_ARCH)
+ifneq (,$(filter $(TARGET_ARCH),x86_64 aarch64 sbsa ppc64le armv7l))
+    ifneq ($(TARGET_ARCH),$(HOST_ARCH))
+        ifneq (,$(filter $(TARGET_ARCH),x86_64 aarch64 sbsa ppc64le))
+            TARGET_SIZE := 64
+        else ifneq (,$(filter $(TARGET_ARCH),armv7l))
+            TARGET_SIZE := 32
+        endif
+    else
+        TARGET_SIZE := $(shell getconf LONG_BIT)
+    endif
+else
+    $(error ERROR - unsupported value $(TARGET_ARCH) for TARGET_ARCH!)
+endif
+
+# sbsa and aarch64 systems look similar. Need to differentiate them at host level for now.
+ifeq ($(HOST_ARCH),aarch64)
+    ifeq ($(CUDA_PATH)/targets/sbsa-linux,$(shell ls -1d $(CUDA_PATH)/targets/sbsa-linux 2>/dev/null))
+        HOST_ARCH := sbsa
+        TARGET_ARCH := sbsa
+    endif
+endif
+
+ifneq ($(TARGET_ARCH),$(HOST_ARCH))
+    ifeq (,$(filter $(HOST_ARCH)-$(TARGET_ARCH),aarch64-armv7l x86_64-armv7l x86_64-aarch64 x86_64-sbsa x86_64-ppc64le))
+        $(error ERROR - cross compiling from $(HOST_ARCH) to $(TARGET_ARCH) is not supported!)
+    endif
+endif
+
+# When on native aarch64 system with userspace of 32-bit, change TARGET_ARCH to armv7l
+ifeq ($(HOST_ARCH)-$(TARGET_ARCH)-$(TARGET_SIZE),aarch64-aarch64-32)
+    TARGET_ARCH = armv7l
+endif
+
+# operating system
+HOST_OS   := $(shell uname -s 2>/dev/null | tr "[:upper:]" "[:lower:]")
+TARGET_OS ?= $(HOST_OS)
+ifeq (,$(filter $(TARGET_OS),linux darwin qnx android))
+    $(error ERROR - unsupported value $(TARGET_OS) for TARGET_OS!)
+endif
+
+# host compiler
+ifeq ($(TARGET_OS),darwin)
+    ifeq ($(shell expr `xcodebuild -version | grep -i xcode | awk '{print $$2}' | cut -d'.' -f1` \>= 5),1)
+        HOST_COMPILER ?= clang++
+    endif
+else ifneq ($(TARGET_ARCH),$(HOST_ARCH))
+    ifeq ($(HOST_ARCH)-$(TARGET_ARCH),x86_64-armv7l)
+        ifeq ($(TARGET_OS),linux)
+            HOST_COMPILER ?= arm-linux-gnueabihf-g++
+        else ifeq ($(TARGET_OS),qnx)
+            ifeq ($(QNX_HOST),)
+                $(error ERROR - QNX_HOST must be passed to the QNX host toolchain)
+            endif
+            ifeq ($(QNX_TARGET),)
+                $(error ERROR - QNX_TARGET must be passed to the QNX target toolchain)
+            endif
+            export QNX_HOST
+            export QNX_TARGET
+            HOST_COMPILER ?= $(QNX_HOST)/usr/bin/arm-unknown-nto-qnx6.6.0eabi-g++
+        else ifeq ($(TARGET_OS),android)
+            HOST_COMPILER ?= arm-linux-androideabi-g++
+        endif
+    else ifeq ($(TARGET_ARCH),aarch64)
+        ifeq ($(TARGET_OS), linux)
+            HOST_COMPILER ?= aarch64-linux-gnu-g++
+        else ifeq ($(TARGET_OS),qnx)
+            ifeq ($(QNX_HOST),)
+                $(error ERROR - QNX_HOST must be passed to the QNX host toolchain)
+            endif
+            ifeq ($(QNX_TARGET),)
+                $(error ERROR - QNX_TARGET must be passed to the QNX target toolchain)
+            endif
+            export QNX_HOST
+            export QNX_TARGET
+            HOST_COMPILER ?= $(QNX_HOST)/usr/bin/q++
+        else ifeq ($(TARGET_OS), android)
+            HOST_COMPILER ?= aarch64-linux-android-clang++
+        endif
+    else ifeq ($(TARGET_ARCH),sbsa)
+        HOST_COMPILER ?= aarch64-linux-gnu-g++
+    else ifeq ($(TARGET_ARCH),ppc64le)
+        HOST_COMPILER ?= powerpc64le-linux-gnu-g++
+    endif
+endif
+HOST_COMPILER ?= g++
+NVCC          := $(CUDA_PATH)/bin/nvcc -ccbin $(HOST_COMPILER)
+
+# internal flags
+NVCCFLAGS   := -m${TARGET_SIZE}
+CCFLAGS     :=
+LDFLAGS     :=
+
+# build flags
+ifeq ($(TARGET_OS),darwin)
+    LDFLAGS += -rpath $(CUDA_PATH)/lib
+    CCFLAGS += -arch $(HOST_ARCH)
+else ifeq ($(HOST_ARCH)-$(TARGET_ARCH)-$(TARGET_OS),x86_64-armv7l-linux)
+    LDFLAGS += --dynamic-linker=/lib/ld-linux-armhf.so.3
+    CCFLAGS += -mfloat-abi=hard
+else ifeq ($(TARGET_OS),android)
+    LDFLAGS += -pie
+    CCFLAGS += -fpie -fpic -fexceptions
+endif
+
+ifneq ($(TARGET_ARCH),$(HOST_ARCH))
+    ifeq ($(TARGET_ARCH)-$(TARGET_OS),armv7l-linux)
+        ifneq ($(TARGET_FS),)
+            GCCVERSIONLTEQ46 := $(shell expr `$(HOST_COMPILER) -dumpversion` \<= 4.6)
+            ifeq ($(GCCVERSIONLTEQ46),1)
+                CCFLAGS += --sysroot=$(TARGET_FS)
+            endif
+            LDFLAGS += --sysroot=$(TARGET_FS)
+            LDFLAGS += -rpath-link=$(TARGET_FS)/lib
+            LDFLAGS += -rpath-link=$(TARGET_FS)/usr/lib
+            LDFLAGS += -rpath-link=$(TARGET_FS)/usr/lib/arm-linux-gnueabihf
+        endif
+    endif
+    ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-linux)
+        ifneq ($(TARGET_FS),)
+            GCCVERSIONLTEQ46 := $(shell expr `$(HOST_COMPILER) -dumpversion` \<= 4.6)
+            ifeq ($(GCCVERSIONLTEQ46),1)
+                CCFLAGS += --sysroot=$(TARGET_FS)
+            endif
+            LDFLAGS += --sysroot=$(TARGET_FS)
+            LDFLAGS += -rpath-link=$(TARGET_FS)/lib -L$(TARGET_FS)/lib
+            LDFLAGS += -rpath-link=$(TARGET_FS)/lib/aarch64-linux-gnu -L$(TARGET_FS)/lib/aarch64-linux-gnu
+            LDFLAGS += -rpath-link=$(TARGET_FS)/usr/lib -L$(TARGET_FS)/usr/lib
+            LDFLAGS += -rpath-link=$(TARGET_FS)/usr/lib/aarch64-linux-gnu -L$(TARGET_FS)/usr/lib/aarch64-linux-gnu
+            LDFLAGS += --unresolved-symbols=ignore-in-shared-libs
+            CCFLAGS += -isystem=$(TARGET_FS)/usr/include -I$(TARGET_FS)/usr/include -I$(TARGET_FS)/usr/include/libdrm
+            CCFLAGS += -isystem=$(TARGET_FS)/usr/include/aarch64-linux-gnu -I$(TARGET_FS)/usr/include/aarch64-linux-gnu
+        endif
+    endif
+    ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-qnx)
+        NVCCFLAGS += --qpp-config 5.4.0,gcc_ntoaarch64le
+        CCFLAGS += -DWIN_INTERFACE_CUSTOM -I/usr/include/aarch64-qnx-gnu
+        LDFLAGS += -lsocket
+        LDFLAGS += -L/usr/lib/aarch64-qnx-gnu
+        CCFLAGS += "-Wl\,-rpath-link\,/usr/lib/aarch64-qnx-gnu"
+        ifdef TARGET_OVERRIDE
+            LDFLAGS += -lslog2
+        endif
+
+        ifneq ($(TARGET_FS),)
+            LDFLAGS += -L$(TARGET_FS)/usr/lib
+            CCFLAGS += "-Wl\,-rpath-link\,$(TARGET_FS)/usr/lib"
+            LDFLAGS += -L$(TARGET_FS)/usr/libnvidia
+            CCFLAGS += "-Wl\,-rpath-link\,$(TARGET_FS)/usr/libnvidia"
+            CCFLAGS += -I$(TARGET_FS)/../include
+        endif
+    endif
+endif
+
+ifdef TARGET_OVERRIDE # cuda toolkit targets override
+    NVCCFLAGS += -target-dir $(TARGET_OVERRIDE)
+endif
+
+# Install directory of different arch
+CUDA_INSTALL_TARGET_DIR :=
+ifeq ($(TARGET_ARCH)-$(TARGET_OS),armv7l-linux)
+    CUDA_INSTALL_TARGET_DIR = targets/armv7-linux-gnueabihf/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-linux)
+    CUDA_INSTALL_TARGET_DIR = targets/aarch64-linux/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),sbsa-linux)
+    CUDA_INSTALL_TARGET_DIR = targets/sbsa-linux/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),armv7l-android)
+    CUDA_INSTALL_TARGET_DIR = targets/armv7-linux-androideabi/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-android)
+    CUDA_INSTALL_TARGET_DIR = targets/aarch64-linux-androideabi/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),armv7l-qnx)
+    CUDA_INSTALL_TARGET_DIR = targets/ARMv7-linux-QNX/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-qnx)
+    CUDA_INSTALL_TARGET_DIR = targets/aarch64-qnx/
+else ifeq ($(TARGET_ARCH),ppc64le)
+    CUDA_INSTALL_TARGET_DIR = targets/ppc64le-linux/
+endif
+
+# Debug build flags
+ifeq ($(dbg),1)
+      NVCCFLAGS += -g -G
+      BUILD_TYPE := debug
+else
+      BUILD_TYPE := release
+endif
+
+ALL_CCFLAGS :=
+ALL_CCFLAGS += $(NVCCFLAGS)
+ALL_CCFLAGS += $(EXTRA_NVCCFLAGS)
+ALL_CCFLAGS += $(addprefix -Xcompiler ,$(CCFLAGS))
+ALL_CCFLAGS += $(addprefix -Xcompiler ,$(EXTRA_CCFLAGS))
+
+SAMPLE_ENABLED := 1
+
+ALL_LDFLAGS :=
+ALL_LDFLAGS += $(ALL_CCFLAGS)
+ALL_LDFLAGS += $(addprefix -Xlinker ,$(LDFLAGS))
+ALL_LDFLAGS += $(addprefix -Xlinker ,$(EXTRA_LDFLAGS))
+
+# Common includes and paths for CUDA
+INCLUDES  := -I../../Common
+LIBRARIES :=
+
+################################################################################
+
+#Detect if installed version of GCC supports required C++11
+ifeq ($(TARGET_OS),linux)
+    empty :=
+    space := $(empty) $(empty)
+    GCCVERSIONSTRING := $(shell expr `$(HOST_COMPILER) -dumpversion`)
+#Create version number without "."
+    GCCVERSION := $(shell expr `echo $(GCCVERSIONSTRING)` | cut -f1 -d.)
+    GCCVERSION += $(shell expr `echo $(GCCVERSIONSTRING)` | cut -f2 -d.)
+    GCCVERSION += $(shell expr `echo $(GCCVERSIONSTRING)` | cut -f3 -d.)
+# Make sure the version number has at least 3 decimals
+    GCCVERSION += 00
+# Remove spaces from the version number
+    GCCVERSION := $(subst $(space),$(empty),$(GCCVERSION))
+#$(warning $(GCCVERSION))
+
+    IS_MIN_VERSION := $(shell expr `echo $(GCCVERSION)` \>= 51000)
+
+    ifeq ($(IS_MIN_VERSION), 1)
+        $(info >>> GCC Version is greater or equal to 5.1.0 <<<)
+    else
+        $(info >>> Waiving build. Minimum GCC version required is 5.1.0<<<)
+        SAMPLE_ENABLED := 0
+    endif
+endif
+
+# Gencode arguments
+ifeq ($(TARGET_ARCH),$(filter $(TARGET_ARCH),armv7l aarch64))
+SMS ?= 35 37 50 52 60 61 70 72 75 80 86
+else
+SMS ?= 35 37 50 52 60 61 70 75 80 86
+endif
+
+ifeq ($(SMS),)
+$(info >>> WARNING - no SM architectures have been specified - waiving sample <<<)
+SAMPLE_ENABLED := 0
+endif
+
+ifeq ($(GENCODE_FLAGS),)
+# Generate SASS code for each SM architecture listed in $(SMS)
+$(foreach sm,$(SMS),$(eval GENCODE_FLAGS += -gencode arch=compute_$(sm),code=sm_$(sm)))
+
+# Generate PTX code from the highest SM architecture in $(SMS) to guarantee forward-compatibility
+HIGHEST_SM := $(lastword $(sort $(SMS)))
+ifneq ($(HIGHEST_SM),)
+GENCODE_FLAGS += -gencode arch=compute_$(HIGHEST_SM),code=compute_$(HIGHEST_SM)
+endif
+endif
+
+ALL_CCFLAGS += --threads 0 --std=c++11
+
+INCLUDES += -Iinc
+
+ifeq ($(SAMPLE_ENABLED),0)
+EXEC ?= @echo "[@]"
+endif
+
+################################################################################
+
+# Target rules
+all: build
+
+build: FDTD3d
+
+check.deps:
+ifeq ($(SAMPLE_ENABLED),0)
+	@echo "Sample will be waived due to the above missing dependencies"
+else
+	@echo "Sample is ready - all dependencies have been met"
+endif
+
+FDTD3d.o:src/FDTD3d.cpp
+	$(EXEC) $(NVCC) $(INCLUDES) $(ALL_CCFLAGS) $(GENCODE_FLAGS) -o $@ -c $<
+
+FDTD3dGPU.o:src/FDTD3dGPU.cu
+	$(EXEC) $(NVCC) $(INCLUDES) $(ALL_CCFLAGS) $(GENCODE_FLAGS) -o $@ -c $<
+
+FDTD3dReference.o:src/FDTD3dReference.cpp
+	$(EXEC) $(NVCC) $(INCLUDES) $(ALL_CCFLAGS) $(GENCODE_FLAGS) -o $@ -c $<
+
+FDTD3d: FDTD3d.o FDTD3dGPU.o FDTD3dReference.o
+	$(EXEC) $(NVCC) $(ALL_LDFLAGS) $(GENCODE_FLAGS) -o $@ $+ $(LIBRARIES)
+	$(EXEC) mkdir -p ../../bin/$(TARGET_ARCH)/$(TARGET_OS)/$(BUILD_TYPE)
+	$(EXEC) cp $@ ../../bin/$(TARGET_ARCH)/$(TARGET_OS)/$(BUILD_TYPE)
+
+run: build
+	$(EXEC) ./FDTD3d
+
+clean:
+	rm -f FDTD3d FDTD3d.o FDTD3dGPU.o FDTD3dReference.o
+	rm -rf ../../bin/$(TARGET_ARCH)/$(TARGET_OS)/$(BUILD_TYPE)/FDTD3d
+
+clobber: clean

Samples/FDTD3d/NsightEclipse.xml

@@ -0,0 +1,71 @@
+<?xml version="1.0" encoding="UTF-8"?> 
+<!DOCTYPE entry SYSTEM "SamplesInfo.dtd">
+<entry>
+  <name>FDTD3d</name>
+  <description><![CDATA[This sample applies a finite differences time domain progression stencil on a 3D surface.]]></description>
+  <devicecompilation>whole</devicecompilation>
+  <includepaths>
+    <path>inc</path>
+    <path>./</path>
+    <path>../</path>
+    <path>../../Common</path>
+  </includepaths>
+  <keyconcepts>
+    <concept level="advanced">Performance Strategies</concept>
+  </keyconcepts>
+  <keywords>
+    <keyword>GPGPU</keyword>
+    <keyword>CUDA</keyword>
+    <keyword>finite difference</keyword>
+    <keyword>fdtd</keyword>
+    <keyword>differential equation</keyword>
+    <keyword>pde</keyword>
+    <keyword>ode</keyword>
+  </keywords>
+  <libraries>
+  </libraries>
+  <librarypaths>
+  </librarypaths>
+  <nsight_eclipse>true</nsight_eclipse>
+  <primary_file>FDTD3d.cpp</primary_file>
+  <scopes>
+    <scope>1:CUDA Advanced Topics</scope>
+    <scope>1:Performance Strategies</scope>
+  </scopes>
+  <sm-arch>sm35</sm-arch>
+  <sm-arch>sm37</sm-arch>
+  <sm-arch>sm50</sm-arch>
+  <sm-arch>sm52</sm-arch>
+  <sm-arch>sm60</sm-arch>
+  <sm-arch>sm61</sm-arch>
+  <sm-arch>sm70</sm-arch>
+  <sm-arch>sm72</sm-arch>
+  <sm-arch>sm75</sm-arch>
+  <sm-arch>sm80</sm-arch>
+  <sm-arch>sm86</sm-arch>
+  <supported_envs>
+    <env>
+      <arch>x86_64</arch>
+      <platform>linux</platform>
+    </env>
+    <env>
+      <platform>windows7</platform>
+    </env>
+    <env>
+      <arch>x86_64</arch>
+      <platform>macosx</platform>
+    </env>
+    <env>
+      <arch>arm</arch>
+    </env>
+    <env>
+      <arch>ppc64le</arch>
+      <platform>linux</platform>
+    </env>
+  </supported_envs>
+  <supported_sm_architectures>
+    <include>all</include>
+  </supported_sm_architectures>
+  <title>CUDA C 3D FDTD</title>
+  <type>exe</type>
+</entry>

Samples/FDTD3d/README.md

@@ -0,0 +1,67 @@
+# FDTD3d - CUDA C 3D FDTD
+
+## Description
+
+This sample applies a finite differences time domain progression stencil on a 3D surface.
+
+## Key Concepts
+
+Performance Strategies
+
+## Supported SM Architectures
+
+[SM 3.5 ](https://developer.nvidia.com/cuda-gpus)  [SM 3.7 ](https://developer.nvidia.com/cuda-gpus)  [SM 5.0 ](https://developer.nvidia.com/cuda-gpus)  [SM 5.2 ](https://developer.nvidia.com/cuda-gpus)  [SM 6.0 ](https://developer.nvidia.com/cuda-gpus)  [SM 6.1 ](https://developer.nvidia.com/cuda-gpus)  [SM 7.0 ](https://developer.nvidia.com/cuda-gpus)  [SM 7.2 ](https://developer.nvidia.com/cuda-gpus)  [SM 7.5 ](https://developer.nvidia.com/cuda-gpus)  [SM 8.0 ](https://developer.nvidia.com/cuda-gpus)  [SM 8.6 ](https://developer.nvidia.com/cuda-gpus)
+
+## Supported OSes
+
+Linux, Windows
+
+## Supported CPU Architecture
+
+x86_64, ppc64le, armv7l
+
+## CUDA APIs involved
+
+## Prerequisites
+
+Download and install the [CUDA Toolkit 11.5](https://developer.nvidia.com/cuda-downloads) for your corresponding platform.
+
+## Build and Run
+
+### Windows
+The Windows samples are built using the Visual Studio IDE. Solution files (.sln) are provided for each supported version of Visual Studio, using the format:
+```
+*_vs<version>.sln - for Visual Studio <version>
+```
+Each individual sample has its own set of solution files in its directory:
+
+To build/examine all the samples at once, the complete solution files should be used. To build/examine a single sample, the individual sample solution files should be used.
+> **Note:** Some samples require that the Microsoft DirectX SDK (June 2010 or newer) be installed and that the VC++ directory paths are properly set up (**Tools > Options...**). Check DirectX Dependencies section for details."
+
+### Linux
+The Linux samples are built using makefiles. To use the makefiles, change the current directory to the sample directory you wish to build, and run make:
+```
+$ cd <sample_dir>
+$ make
+```
+The samples makefiles can take advantage of certain options:
+*  **TARGET_ARCH=<arch>** - cross-compile targeting a specific architecture. Allowed architectures are x86_64, ppc64le, armv7l.
+    By default, TARGET_ARCH is set to HOST_ARCH. On a x86_64 machine, not setting TARGET_ARCH is the equivalent of setting TARGET_ARCH=x86_64.<br/>
+`$ make TARGET_ARCH=x86_64` <br/> `$ make TARGET_ARCH=ppc64le` <br/> `$ make TARGET_ARCH=armv7l` <br/>
+    See [here](http://docs.nvidia.com/cuda/cuda-samples/index.html#cross-samples) for more details.
+*   **dbg=1** - build with debug symbols
+    ```
+    $ make dbg=1
+    ```
+*   **SMS="A B ..."** - override the SM architectures for which the sample will be built, where `"A B ..."` is a space-delimited list of SM architectures. For example, to generate SASS for SM 50 and SM 60, use `SMS="50 60"`.
+    ```
+    $ make SMS="50 60"
+    ```
+
+*  **HOST_COMPILER=<host_compiler>** - override the default g++ host compiler. See the [Linux Installation Guide](http://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html#system-requirements) for a list of supported host compilers.
+```
+    $ make HOST_COMPILER=g++
+```
+
+## References (for more details)
+

Samples/FDTD3d/inc/FDTD3d.h

@@ -0,0 +1,54 @@
+/* Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _FDTD3D_H_
+#define _FDTD3D_H_
+
+// The values are set to give reasonable runtimes, they can
+// be changed but note that running very large dimensions can
+// take a very long time and you should avoid running on your
+// primary display in this case.
+#define k_dim_min 96
+#define k_dim_max 376
+#define k_dim_qa 248
+
+// Note that the radius is defined here as exactly 4 since the
+// kernel code uses a constant. If you want a different radius
+// you must change the kernel accordingly.
+#define k_radius_min 4
+#define k_radius_max 4
+#define k_radius_default 4
+
+// The values are set to give reasonable runtimes, they can
+// be changed but note that running a very large number of
+// timesteps can take a very long time and you should avoid
+// running on your primary display in this case.
+#define k_timesteps_min 1
+#define k_timesteps_max 10
+#define k_timesteps_default 5
+
+#endif

Samples/FDTD3d/inc/FDTD3dGPU.h

@@ -0,0 +1,51 @@
+/* Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _FDTD3DGPU_H_
+#define _FDTD3DGPU_H_
+
+#include <cstddef>
+#if defined(WIN32) || defined(_WIN32) || defined(WIN64) || \
+    defined(_WIN64) && defined(_MSC_VER)
+typedef unsigned __int64 memsize_t;
+#else
+#include <stdint.h>
+typedef uint64_t memsize_t;
+#endif
+
+#define k_blockDimX 32
+#define k_blockDimMaxY 16
+#define k_blockSizeMin 128
+#define k_blockSizeMax (k_blockDimX * k_blockDimMaxY)
+
+bool getTargetDeviceGlobalMemSize(memsize_t *result, const int argc,
+                                  const char **argv);
+bool fdtdGPU(float *output, const float *input, const float *coeff,
+             const int dimx, const int dimy, const int dimz, const int radius,
+             const int timesteps, const int argc, const char **argv);
+
+#endif

Samples/FDTD3d/inc/FDTD3dGPUKernel.cuh

@@ -0,0 +1,150 @@
+/* Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "FDTD3dGPU.h"
+#include <cooperative_groups.h>
+
+namespace cg = cooperative_groups;
+
+// Note: If you change the RADIUS, you should also change the unrolling below
+#define RADIUS 4
+
+__constant__ float stencil[RADIUS + 1];
+
+__global__ void FiniteDifferencesKernel(float *output, const float *input,
+                                        const int dimx, const int dimy,
+                                        const int dimz) {
+  bool validr = true;
+  bool validw = true;
+  const int gtidx = blockIdx.x * blockDim.x + threadIdx.x;
+  const int gtidy = blockIdx.y * blockDim.y + threadIdx.y;
+  const int ltidx = threadIdx.x;
+  const int ltidy = threadIdx.y;
+  const int workx = blockDim.x;
+  const int worky = blockDim.y;
+  // Handle to thread block group
+  cg::thread_block cta = cg::this_thread_block();
+  __shared__ float tile[k_blockDimMaxY + 2 * RADIUS][k_blockDimX + 2 * RADIUS];
+
+  const int stride_y = dimx + 2 * RADIUS;
+  const int stride_z = stride_y * (dimy + 2 * RADIUS);
+
+  int inputIndex = 0;
+  int outputIndex = 0;
+
+  // Advance inputIndex to start of inner volume
+  inputIndex += RADIUS * stride_y + RADIUS;
+
+  // Advance inputIndex to target element
+  inputIndex += gtidy * stride_y + gtidx;
+
+  float infront[RADIUS];
+  float behind[RADIUS];
+  float current;
+
+  const int tx = ltidx + RADIUS;
+  const int ty = ltidy + RADIUS;
+
+  // Check in bounds
+  if ((gtidx >= dimx + RADIUS) || (gtidy >= dimy + RADIUS)) validr = false;
+
+  if ((gtidx >= dimx) || (gtidy >= dimy)) validw = false;
+
+  // Preload the "infront" and "behind" data
+  for (int i = RADIUS - 2; i >= 0; i--) {
+    if (validr) behind[i] = input[inputIndex];
+
+    inputIndex += stride_z;
+  }
+
+  if (validr) current = input[inputIndex];
+
+  outputIndex = inputIndex;
+  inputIndex += stride_z;
+
+  for (int i = 0; i < RADIUS; i++) {
+    if (validr) infront[i] = input[inputIndex];
+
+    inputIndex += stride_z;
+  }
+
+// Step through the xy-planes
+#pragma unroll 9
+
+  for (int iz = 0; iz < dimz; iz++) {
+    // Advance the slice (move the thread-front)
+    for (int i = RADIUS - 1; i > 0; i--) behind[i] = behind[i - 1];
+
+    behind[0] = current;
+    current = infront[0];
+#pragma unroll 4
+
+    for (int i = 0; i < RADIUS - 1; i++) infront[i] = infront[i + 1];
+
+    if (validr) infront[RADIUS - 1] = input[inputIndex];
+
+    inputIndex += stride_z;
+    outputIndex += stride_z;
+    cg::sync(cta);
+
+    // Note that for the work items on the boundary of the problem, the
+    // supplied index when reading the halo (below) may wrap to the
+    // previous/next row or even the previous/next xy-plane. This is
+    // acceptable since a) we disable the output write for these work
+    // items and b) there is at least one xy-plane before/after the
+    // current plane, so the access will be within bounds.
+
+    // Update the data slice in the local tile
+    // Halo above & below
+    if (ltidy < RADIUS) {
+      tile[ltidy][tx] = input[outputIndex - RADIUS * stride_y];
+      tile[ltidy + worky + RADIUS][tx] = input[outputIndex + worky * stride_y];
+    }
+
+    // Halo left & right
+    if (ltidx < RADIUS) {
+      tile[ty][ltidx] = input[outputIndex - RADIUS];
+      tile[ty][ltidx + workx + RADIUS] = input[outputIndex + workx];
+    }
+
+    tile[ty][tx] = current;
+    cg::sync(cta);
+
+    // Compute the output value
+    float value = stencil[0] * current;
+#pragma unroll 4
+
+    for (int i = 1; i <= RADIUS; i++) {
+      value +=
+          stencil[i] * (infront[i - 1] + behind[i - 1] + tile[ty - i][tx] +
+                        tile[ty + i][tx] + tile[ty][tx - i] + tile[ty][tx + i]);
+    }
+
+    // Store the output value
+    if (validw) output[outputIndex] = value;
+  }
+}

Samples/FDTD3d/inc/FDTD3dReference.h

@@ -0,0 +1,44 @@
+/* Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _FDTD3DREFERENCE_H_
+#define _FDTD3DREFERENCE_H_
+
+void generateRandomData(float *data, const int dimx, const int dimy,
+                        const int dimz, const float lowerBound,
+                        const float upperBound);
+void generatePatternData(float *data, const int dimx, const int dimy,
+                         const int dimz, const float lowerBound,
+                         const float upperBound);
+bool fdtdReference(float *output, const float *input, const float *coeff,
+                   const int dimx, const int dimy, const int dimz,
+                   const int radius, const int timesteps);
+bool compareData(const float *output, const float *reference, const int dimx,
+                 const int dimy, const int dimz, const int radius,
+                 const float tolerance = 0.0001f);
+
+#endif

Samples/FDTD3d/src/FDTD3d.cpp

@@ -0,0 +1,235 @@
+/* Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "FDTD3d.h"
+
+#include <iostream>
+#include <iomanip>
+
+#include "FDTD3dReference.h"
+#include "FDTD3dGPU.h"
+
+#include <helper_functions.h>
+
+#include <math.h>
+#include <assert.h>
+
+#ifndef CLAMP
+#define CLAMP(a, min, max) (MIN(max, MAX(a, min)))
+#endif
+
+//// Name of the log file
+// const char *printfFile = "FDTD3d.txt";
+
+// Forward declarations
+bool runTest(int argc, const char **argv);
+void showHelp(const int argc, const char **argv);
+
+int main(int argc, char **argv) {
+  bool bTestResult = false;
+  // Start the log
+  printf("%s Starting...\n\n", argv[0]);
+
+  // Check help flag
+  if (checkCmdLineFlag(argc, (const char **)argv, "help")) {
+    printf("Displaying help on console\n");
+    showHelp(argc, (const char **)argv);
+    bTestResult = true;
+  } else {
+    // Execute
+    bTestResult = runTest(argc, (const char **)argv);
+  }
+
+  // Finish
+  exit(bTestResult ? EXIT_SUCCESS : EXIT_FAILURE);
+}
+
+void showHelp(const int argc, const char **argv) {
+  if (argc > 0) std::cout << std::endl << argv[0] << std::endl;
+
+  std::cout << std::endl << "Syntax:" << std::endl;
+  std::cout << std::left;
+  std::cout << "    " << std::setw(20) << "--device=<device>"
+            << "Specify device to use for execution" << std::endl;
+  std::cout << "    " << std::setw(20) << "--dimx=<N>"
+            << "Specify number of elements in x direction (excluding halo)"
+            << std::endl;
+  std::cout << "    " << std::setw(20) << "--dimy=<N>"
+            << "Specify number of elements in y direction (excluding halo)"
+            << std::endl;
+  std::cout << "    " << std::setw(20) << "--dimz=<N>"
+            << "Specify number of elements in z direction (excluding halo)"
+            << std::endl;
+  std::cout << "    " << std::setw(20) << "--radius=<N>"
+            << "Specify radius of stencil" << std::endl;
+  std::cout << "    " << std::setw(20) << "--timesteps=<N>"
+            << "Specify number of timesteps" << std::endl;
+  std::cout << "    " << std::setw(20) << "--block-size=<N>"
+            << "Specify number of threads per block" << std::endl;
+  std::cout << std::endl;
+  std::cout << "    " << std::setw(20) << "--noprompt"
+            << "Skip prompt before exit" << std::endl;
+  std::cout << std::endl;
+}
+
+bool runTest(int argc, const char **argv) {
+  float *host_output;
+  float *device_output;
+  float *input;
+  float *coeff;
+
+  int defaultDim;
+  int dimx;
+  int dimy;
+  int dimz;
+  int outerDimx;
+  int outerDimy;
+  int outerDimz;
+  int radius;
+  int timesteps;
+  size_t volumeSize;
+  memsize_t memsize;
+
+  const float lowerBound = 0.0f;
+  const float upperBound = 1.0f;
+
+  // Determine default dimensions
+  printf("Set-up, based upon target device GMEM size...\n");
+  // Get the memory size of the target device
+  printf(" getTargetDeviceGlobalMemSize\n");
+  getTargetDeviceGlobalMemSize(&memsize, argc, argv);
+
+  // We can never use all the memory so to keep things simple we aim to
+  // use around half the total memory
+  memsize /= 2;
+
+  // Most of our memory use is taken up by the input and output buffers -
+  // two buffers of equal size - and for simplicity the volume is a cube:
+  //   dim = floor( (N/2)^(1/3) )
+  defaultDim = (int)floor(pow((memsize / (2.0 * sizeof(float))), 1.0 / 3.0));
+
+  // By default, make the volume edge size an integer multiple of 128B to
+  // improve performance by coalescing memory accesses, in a real
+  // application it would make sense to pad the lines accordingly
+  int roundTarget = 128 / sizeof(float);
+  defaultDim = defaultDim / roundTarget * roundTarget;
+  defaultDim -= k_radius_default * 2;
+
+  // Check dimension is valid
+  if (defaultDim < k_dim_min) {
+    printf(
+        "insufficient device memory (maximum volume on device is %d, must be "
+        "between %d and %d).\n",
+        defaultDim, k_dim_min, k_dim_max);
+    exit(EXIT_FAILURE);
+  } else if (defaultDim > k_dim_max) {
+    defaultDim = k_dim_max;
+  }
+
+  // For QA testing, override default volume size
+  if (checkCmdLineFlag(argc, argv, "qatest")) {
+    defaultDim = MIN(defaultDim, k_dim_qa);
+  }
+
+  // set default dim
+  dimx = defaultDim;
+  dimy = defaultDim;
+  dimz = defaultDim;
+  radius = k_radius_default;
+  timesteps = k_timesteps_default;
+
+  // Parse command line arguments
+  if (checkCmdLineFlag(argc, argv, "dimx")) {
+    dimx =
+        CLAMP(getCmdLineArgumentInt(argc, argv, "dimx"), k_dim_min, k_dim_max);
+  }
+
+  if (checkCmdLineFlag(argc, argv, "dimy")) {
+    dimy =
+        CLAMP(getCmdLineArgumentInt(argc, argv, "dimy"), k_dim_min, k_dim_max);
+  }
+
+  if (checkCmdLineFlag(argc, argv, "dimz")) {
+    dimz =
+        CLAMP(getCmdLineArgumentInt(argc, argv, "dimz"), k_dim_min, k_dim_max);
+  }
+
+  if (checkCmdLineFlag(argc, argv, "radius")) {
+    radius = CLAMP(getCmdLineArgumentInt(argc, argv, "radius"), k_radius_min,
+                   k_radius_max);
+  }
+
+  if (checkCmdLineFlag(argc, argv, "timesteps")) {
+    timesteps = CLAMP(getCmdLineArgumentInt(argc, argv, "timesteps"),
+                      k_timesteps_min, k_timesteps_max);
+  }
+
+  // Determine volume size
+  outerDimx = dimx + 2 * radius;
+  outerDimy = dimy + 2 * radius;
+  outerDimz = dimz + 2 * radius;
+  volumeSize = outerDimx * outerDimy * outerDimz;
+
+  // Allocate memory
+  host_output = (float *)calloc(volumeSize, sizeof(float));
+  input = (float *)malloc(volumeSize * sizeof(float));
+  coeff = (float *)malloc((radius + 1) * sizeof(float));
+
+  // Create coefficients
+  for (int i = 0; i <= radius; i++) {
+    coeff[i] = 0.1f;
+  }
+
+  // Generate data
+  printf(" generateRandomData\n\n");
+  generateRandomData(input, outerDimx, outerDimy, outerDimz, lowerBound,
+                     upperBound);
+  printf(
+      "FDTD on %d x %d x %d volume with symmetric filter radius %d for %d "
+      "timesteps...\n\n",
+      dimx, dimy, dimz, radius, timesteps);
+
+  // Execute on the host
+  printf("fdtdReference...\n");
+  fdtdReference(host_output, input, coeff, dimx, dimy, dimz, radius, timesteps);
+  printf("fdtdReference complete\n");
+
+  // Allocate memory
+  device_output = (float *)calloc(volumeSize, sizeof(float));
+
+  // Execute on the device
+  printf("fdtdGPU...\n");
+  fdtdGPU(device_output, input, coeff, dimx, dimy, dimz, radius, timesteps,
+          argc, argv);
+  printf("fdtdGPU complete\n");
+
+  // Compare the results
+  float tolerance = 0.0001f;
+  printf("\nCompareData (tolerance %f)...\n", tolerance);
+  return compareData(device_output, host_output, dimx, dimy, dimz, radius,
+                     tolerance);
+}

Samples/FDTD3d/src/FDTD3dGPU.cu

@@ -0,0 +1,263 @@
+/* Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "FDTD3dGPU.h"
+
+#include <iostream>
+#include <algorithm>
+#include <helper_functions.h>
+#include <helper_cuda.h>
+
+#include "FDTD3dGPUKernel.cuh"
+
+bool getTargetDeviceGlobalMemSize(memsize_t *result, const int argc,
+                                  const char **argv) {
+  int deviceCount = 0;
+  int targetDevice = 0;
+  size_t memsize = 0;
+
+  // Get the number of CUDA enabled GPU devices
+  printf(" cudaGetDeviceCount\n");
+  checkCudaErrors(cudaGetDeviceCount(&deviceCount));
+
+  // Select target device (device 0 by default)
+  targetDevice = findCudaDevice(argc, (const char **)argv);
+
+  // Query target device for maximum memory allocation
+  printf(" cudaGetDeviceProperties\n");
+  struct cudaDeviceProp deviceProp;
+  checkCudaErrors(cudaGetDeviceProperties(&deviceProp, targetDevice));
+
+  memsize = deviceProp.totalGlobalMem;
+
+  // Save the result
+  *result = (memsize_t)memsize;
+  return true;
+}
+
+bool fdtdGPU(float *output, const float *input, const float *coeff,
+             const int dimx, const int dimy, const int dimz, const int radius,
+             const int timesteps, const int argc, const char **argv) {
+  const int outerDimx = dimx + 2 * radius;
+  const int outerDimy = dimy + 2 * radius;
+  const int outerDimz = dimz + 2 * radius;
+  const size_t volumeSize = outerDimx * outerDimy * outerDimz;
+  int deviceCount = 0;
+  int targetDevice = 0;
+  float *bufferOut = 0;
+  float *bufferIn = 0;
+  dim3 dimBlock;
+  dim3 dimGrid;
+
+  // Ensure that the inner data starts on a 128B boundary
+  const int padding = (128 / sizeof(float)) - radius;
+  const size_t paddedVolumeSize = volumeSize + padding;
+
+#ifdef GPU_PROFILING
+  cudaEvent_t profileStart = 0;
+  cudaEvent_t profileEnd = 0;
+  const int profileTimesteps = timesteps - 1;
+
+  if (profileTimesteps < 1) {
+    printf(
+        " cannot profile with fewer than two timesteps (timesteps=%d), "
+        "profiling is disabled.\n",
+        timesteps);
+  }
+
+#endif
+
+  // Check the radius is valid
+  if (radius != RADIUS) {
+    printf("radius is invalid, must be %d - see kernel for details.\n", RADIUS);
+    exit(EXIT_FAILURE);
+  }
+
+  // Get the number of CUDA enabled GPU devices
+  checkCudaErrors(cudaGetDeviceCount(&deviceCount));
+
+  // Select target device (device 0 by default)
+  targetDevice = findCudaDevice(argc, (const char **)argv);
+
+  checkCudaErrors(cudaSetDevice(targetDevice));
+
+  // Allocate memory buffers
+  checkCudaErrors(
+      cudaMalloc((void **)&bufferOut, paddedVolumeSize * sizeof(float)));
+  checkCudaErrors(
+      cudaMalloc((void **)&bufferIn, paddedVolumeSize * sizeof(float)));
+
+  // Check for a command-line specified block size
+  int userBlockSize;
+
+  if (checkCmdLineFlag(argc, (const char **)argv, "block-size")) {
+    userBlockSize = getCmdLineArgumentInt(argc, argv, "block-size");
+    // Constrain to a multiple of k_blockDimX
+    userBlockSize = (userBlockSize / k_blockDimX * k_blockDimX);
+
+    // Constrain within allowed bounds
+    userBlockSize = MIN(MAX(userBlockSize, k_blockSizeMin), k_blockSizeMax);
+  } else {
+    userBlockSize = k_blockSizeMax;
+  }
+
+  // Check the device limit on the number of threads
+  struct cudaFuncAttributes funcAttrib;
+  checkCudaErrors(cudaFuncGetAttributes(&funcAttrib, FiniteDifferencesKernel));
+
+  userBlockSize = MIN(userBlockSize, funcAttrib.maxThreadsPerBlock);
+
+  // Set the block size
+  dimBlock.x = k_blockDimX;
+  // Visual Studio 2005 does not like std::min
+  //    dimBlock.y = std::min<size_t>(userBlockSize / k_blockDimX,
+  //    (size_t)k_blockDimMaxY);
+  dimBlock.y = ((userBlockSize / k_blockDimX) < (size_t)k_blockDimMaxY)
+                   ? (userBlockSize / k_blockDimX)
+                   : (size_t)k_blockDimMaxY;
+  dimGrid.x = (unsigned int)ceil((float)dimx / dimBlock.x);
+  dimGrid.y = (unsigned int)ceil((float)dimy / dimBlock.y);
+  printf(" set block size to %dx%d\n", dimBlock.x, dimBlock.y);
+  printf(" set grid size to %dx%d\n", dimGrid.x, dimGrid.y);
+
+  // Check the block size is valid
+  if (dimBlock.x < RADIUS || dimBlock.y < RADIUS) {
+    printf("invalid block size, x (%d) and y (%d) must be >= radius (%d).\n",
+           dimBlock.x, dimBlock.y, RADIUS);
+    exit(EXIT_FAILURE);
+  }
+
+  // Copy the input to the device input buffer
+  checkCudaErrors(cudaMemcpy(bufferIn + padding, input,
+                             volumeSize * sizeof(float),
+                             cudaMemcpyHostToDevice));
+
+  // Copy the input to the device output buffer (actually only need the halo)
+  checkCudaErrors(cudaMemcpy(bufferOut + padding, input,
+                             volumeSize * sizeof(float),
+                             cudaMemcpyHostToDevice));
+
+  // Copy the coefficients to the device coefficient buffer
+  checkCudaErrors(
+      cudaMemcpyToSymbol(stencil, (void *)coeff, (radius + 1) * sizeof(float)));
+
+#ifdef GPU_PROFILING
+
+  // Create the events
+  checkCudaErrors(cudaEventCreate(&profileStart));
+  checkCudaErrors(cudaEventCreate(&profileEnd));
+
+#endif
+
+  // Execute the FDTD
+  float *bufferSrc = bufferIn + padding;
+  float *bufferDst = bufferOut + padding;
+  printf(" GPU FDTD loop\n");
+
+#ifdef GPU_PROFILING
+  // Enqueue start event
+  checkCudaErrors(cudaEventRecord(profileStart, 0));
+#endif
+
+  for (int it = 0; it < timesteps; it++) {
+    printf("\tt = %d ", it);
+
+    // Launch the kernel
+    printf("launch kernel\n");
+    FiniteDifferencesKernel<<<dimGrid, dimBlock>>>(bufferDst, bufferSrc, dimx,
+                                                   dimy, dimz);
+
+    // Toggle the buffers
+    // Visual Studio 2005 does not like std::swap
+    //    std::swap<float *>(bufferSrc, bufferDst);
+    float *tmp = bufferDst;
+    bufferDst = bufferSrc;
+    bufferSrc = tmp;
+  }
+
+  printf("\n");
+
+#ifdef GPU_PROFILING
+  // Enqueue end event
+  checkCudaErrors(cudaEventRecord(profileEnd, 0));
+#endif
+
+  // Wait for the kernel to complete
+  checkCudaErrors(cudaDeviceSynchronize());
+
+  // Read the result back, result is in bufferSrc (after final toggle)
+  checkCudaErrors(cudaMemcpy(output, bufferSrc, volumeSize * sizeof(float),
+                             cudaMemcpyDeviceToHost));
+
+// Report time
+#ifdef GPU_PROFILING
+  float elapsedTimeMS = 0;
+
+  if (profileTimesteps > 0) {
+    checkCudaErrors(
+        cudaEventElapsedTime(&elapsedTimeMS, profileStart, profileEnd));
+  }
+
+  if (profileTimesteps > 0) {
+    // Convert milliseconds to seconds
+    double elapsedTime = elapsedTimeMS * 1.0e-3;
+    double avgElapsedTime = elapsedTime / (double)profileTimesteps;
+    // Determine number of computations per timestep
+    size_t pointsComputed = dimx * dimy * dimz;
+    // Determine throughput
+    double throughputM = 1.0e-6 * (double)pointsComputed / avgElapsedTime;
+    printf(
+        "FDTD3d, Throughput = %.4f MPoints/s, Time = %.5f s, Size = %u Points, "
+        "NumDevsUsed = %u, Blocksize = %u\n",
+        throughputM, avgElapsedTime, pointsComputed, 1,
+        dimBlock.x * dimBlock.y);
+  }
+
+#endif
+
+  // Cleanup
+  if (bufferIn) {
+    checkCudaErrors(cudaFree(bufferIn));
+  }
+
+  if (bufferOut) {
+    checkCudaErrors(cudaFree(bufferOut));
+  }
+
+#ifdef GPU_PROFILING
+
+  if (profileStart) {
+    checkCudaErrors(cudaEventDestroy(profileStart));
+  }
+
+  if (profileEnd) {
+    checkCudaErrors(cudaEventDestroy(profileEnd));
+  }
+
+#endif
+  return true;
+}

Samples/FDTD3d/src/FDTD3dReference.cpp

@@ -0,0 +1,178 @@
+/* Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "FDTD3dReference.h"
+
+#include <cstdlib>
+#include <cmath>
+#include <iostream>
+#include <iomanip>
+#include <stdio.h>
+
+void generateRandomData(float *data, const int dimx, const int dimy,
+                        const int dimz, const float lowerBound,
+                        const float upperBound) {
+  srand(0);
+
+  for (int iz = 0; iz < dimz; iz++) {
+    for (int iy = 0; iy < dimy; iy++) {
+      for (int ix = 0; ix < dimx; ix++) {
+        *data = (float)(lowerBound +
+                        ((float)rand() / (float)RAND_MAX) *
+                            (upperBound - lowerBound));
+        ++data;
+      }
+    }
+  }
+}
+
+void generatePatternData(float *data, const int dimx, const int dimy,
+                         const int dimz, const float lowerBound,
+                         const float upperBound) {
+  for (int iz = 0; iz < dimz; iz++) {
+    for (int iy = 0; iy < dimy; iy++) {
+      for (int ix = 0; ix < dimx; ix++) {
+        *data = (float)(lowerBound +
+                        ((float)iz / (float)dimz) * (upperBound - lowerBound));
+        ++data;
+      }
+    }
+  }
+}
+
+bool fdtdReference(float *output, const float *input, const float *coeff,
+                   const int dimx, const int dimy, const int dimz,
+                   const int radius, const int timesteps) {
+  const int outerDimx = dimx + 2 * radius;
+  const int outerDimy = dimy + 2 * radius;
+  const int outerDimz = dimz + 2 * radius;
+  const size_t volumeSize = outerDimx * outerDimy * outerDimz;
+  const int stride_y = outerDimx;
+  const int stride_z = stride_y * outerDimy;
+  float *intermediate = 0;
+  const float *bufsrc = 0;
+  float *bufdst = 0;
+  float *bufdstnext = 0;
+
+  // Allocate temporary buffer
+  printf(" calloc intermediate\n");
+  intermediate = (float *)calloc(volumeSize, sizeof(float));
+
+  // Decide which buffer to use first (result should end up in output)
+  if ((timesteps % 2) == 0) {
+    bufsrc = input;
+    bufdst = intermediate;
+    bufdstnext = output;
+  } else {
+    bufsrc = input;
+    bufdst = output;
+    bufdstnext = intermediate;
+  }
+
+  // Run the FDTD (naive method)
+  printf(" Host FDTD loop\n");
+
+  for (int it = 0; it < timesteps; it++) {
+    printf("\tt = %d\n", it);
+    const float *src = bufsrc;
+    float *dst = bufdst;
+
+    for (int iz = -radius; iz < dimz + radius; iz++) {
+      for (int iy = -radius; iy < dimy + radius; iy++) {
+        for (int ix = -radius; ix < dimx + radius; ix++) {
+          if (ix >= 0 && ix < dimx && iy >= 0 && iy < dimy && iz >= 0 &&
+              iz < dimz) {
+            float value = (*src) * coeff[0];
+
+            for (int ir = 1; ir <= radius; ir++) {
+              value += coeff[ir] * (*(src + ir) + *(src - ir));  // horizontal
+              value += coeff[ir] * (*(src + ir * stride_y) +
+                                    *(src - ir * stride_y));  // vertical
+              value +=
+                  coeff[ir] * (*(src + ir * stride_z) +
+                               *(src - ir * stride_z));  // in front & behind
+            }
+
+            *dst = value;
+          } else {
+            *dst = *src;
+          }
+
+          ++dst;
+          ++src;
+        }
+      }
+    }
+
+    // Rotate buffers
+    float *tmp = bufdst;
+    bufdst = bufdstnext;
+    bufdstnext = tmp;
+    bufsrc = (const float *)tmp;
+  }
+
+  printf("\n");
+
+  if (intermediate) free(intermediate);
+
+  return true;
+}
+
+bool compareData(const float *output, const float *reference, const int dimx,
+                 const int dimy, const int dimz, const int radius,
+                 const float tolerance) {
+  for (int iz = -radius; iz < dimz + radius; iz++) {
+    for (int iy = -radius; iy < dimy + radius; iy++) {
+      for (int ix = -radius; ix < dimx + radius; ix++) {
+        if (ix >= 0 && ix < dimx && iy >= 0 && iy < dimy && iz >= 0 &&
+            iz < dimz) {
+          // Determine the absolute difference
+          float difference = fabs(*reference - *output);
+          float error;
+
+          // Determine the relative error
+          if (*reference != 0)
+            error = difference / *reference;
+          else
+            error = difference;
+
+          // Check the error is within the tolerance
+          if (error > tolerance) {
+            printf("Data error at point (%d,%d,%d)\t%f instead of %f\n", ix, iy,
+                   iz, *output, *reference);
+            return false;
+          }
+        }
+
+        ++output;
+        ++reference;
+      }
+    }
+  }
+
+  return true;
+}

Samples/FilterBorderControlNPP/.vscode/c_cpp_properties.json

@@ -0,0 +1,18 @@
+{
+    "configurations": [
+        {
+            "name": "Linux",
+            "includePath": [
+                "${workspaceFolder}/**",
+                "${workspaceFolder}/../../Common"
+            ],
+            "defines": [],
+            "compilerPath": "/usr/local/cuda/bin/nvcc",
+            "cStandard": "gnu17",
+            "cppStandard": "gnu++14",
+            "intelliSenseMode": "linux-gcc-x64",
+            "configurationProvider": "ms-vscode.makefile-tools"
+        }
+    ],
+    "version": 4
+}

Samples/FilterBorderControlNPP/.vscode/extensions.json

@@ -0,0 +1,7 @@
+{
+    "recommendations": [
+        "nvidia.nsight-vscode-edition",
+        "ms-vscode.cpptools",
+        "ms-vscode.makefile-tools"
+    ]
+}

Samples/FilterBorderControlNPP/.vscode/launch.json

@@ -0,0 +1,10 @@
+{
+    "configurations": [
+        {
+            "name": "CUDA C++: Launch",
+            "type": "cuda-gdb",
+            "request": "launch",
+            "program": "${workspaceFolder}/FilterBorderControlNPP"
+        }
+    ]
+}

Samples/FilterBorderControlNPP/.vscode/tasks.json

@@ -0,0 +1,15 @@
+{
+    "version": "2.0.0",
+    "tasks": [
+        {
+            "label": "sample",
+            "type": "shell",
+            "command": "make dbg=1",
+            "problemMatcher": ["$nvcc"],
+            "group": {
+                "kind": "build",
+                "isDefault": true
+            }
+        }
+    ]
+}

Samples/FilterBorderControlNPP/FilterBorderControlNPP.cpp

@@ -0,0 +1,502 @@
+/* Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+// USING NPP IMAGE FILTER SOURCE IMAGE BORDER CONTROL
+// This sample demonstrates how any border version of an NPP filtering function
+// can be used in the most common mode, with border control enabled. Mentioned
+// functions can be used to duplicate the results of the equivalent non-border
+// version of the NPP functions. They can be also used for enabling and
+// disabling border control on various source image edges depending on what
+// portion of the source image is being used as input.
+
+#if defined(WIN32) || defined(_WIN32) || defined(WIN64) || defined(_WIN64)
+#define WINDOWS_LEAN_AND_MEAN
+#define NOMINMAX
+#include <windows.h>
+#pragma warning(disable : 4819)
+#endif
+
+#include <Exceptions.h>
+#include <ImageIO.h>
+#include <ImagesCPU.h>
+#include <ImagesNPP.h>
+#include <cuda_runtime.h>
+#include <helper_cuda.h>
+#include <helper_string.h>
+#include <npp.h>
+#include <string.h>
+
+#include <fstream>
+#include <iostream>
+
+inline int cudaDeviceInit(int argc, const char **argv) {
+  int deviceCount;
+  checkCudaErrors(cudaGetDeviceCount(&deviceCount));
+
+  if (deviceCount == 0) {
+    std::cerr << "CUDA error: no devices supporting CUDA." << std::endl;
+    exit(EXIT_FAILURE);
+  }
+
+  int dev = findCudaDevice(argc, argv);
+
+  cudaDeviceProp deviceProp;
+  cudaGetDeviceProperties(&deviceProp, dev);
+  std::cerr << "cudaSetDevice GPU" << dev << " = " << deviceProp.name
+            << std::endl;
+
+  checkCudaErrors(cudaSetDevice(dev));
+
+  return dev;
+}
+
+bool printfNPPinfo(int argc, char *argv[]) {
+  const NppLibraryVersion *libVer = nppGetLibVersion();
+
+  printf("NPP Library Version %d.%d.%d\n", libVer->major, libVer->minor,
+         libVer->build);
+
+  int driverVersion, runtimeVersion;
+  cudaDriverGetVersion(&driverVersion);
+  cudaRuntimeGetVersion(&runtimeVersion);
+
+  printf("  CUDA Driver  Version: %d.%d\n", driverVersion / 1000,
+         (driverVersion % 100) / 10);
+  printf("  CUDA Runtime Version: %d.%d\n", runtimeVersion / 1000,
+         (runtimeVersion % 100) / 10);
+
+  // Min spec is SM 1.0 devices
+  bool bVal = checkCudaCapabilities(1, 0);
+  return bVal;
+}
+
+int main(int argc, char *argv[]) {
+  printf("%s Starting...\n\n", argv[0]);
+
+  try {
+    const char *inputFile = "Lena.pgm";
+    std::string sFilename = inputFile;
+    std::string sOutputDir = "./";
+
+    cudaDeviceInit(argc, (const char **)argv);
+
+    if (printfNPPinfo(argc, argv) == false) {
+      cudaDeviceReset();
+      exit(EXIT_SUCCESS);
+    }
+
+    char *filePath;
+
+    if (checkCmdLineFlag(argc, (const char **)argv, "input")) {
+      getCmdLineArgumentString(argc, (const char **)argv, "input", &filePath);
+    } else {
+      filePath = sdkFindFilePath(inputFile, argv[0]);
+    }
+
+    if (!filePath) {
+      std::cerr << "Couldn't find input file " << sFilename << std::endl;
+      exit(1);
+    }
+
+    sFilename = filePath;
+
+    // if we specify the filename at the command line, then we only test
+    // sFilename[0].
+    int file_errors = 0;
+    std::ifstream infile(sFilename.data(), std::ifstream::in);
+
+    if (infile.good()) {
+      std::cout << "gradientFilterBorderNPP opened <" << sFilename.data()
+                << "> successfully!" << std::endl;
+      file_errors = 0;
+      infile.close();
+    } else {
+      std::cout << "gradientFilterBorderNPP unable to open <"
+                << sFilename.data() << ">" << std::endl;
+      file_errors++;
+      infile.close();
+    }
+
+    if (file_errors > 0) {
+      cudaDeviceReset();
+      exit(EXIT_FAILURE);
+    }
+
+    std::string sResultBaseFilename = sFilename;
+
+    std::string::size_type dot = sResultBaseFilename.rfind('.');
+
+    if (dot != std::string::npos) {
+      sResultBaseFilename = sResultBaseFilename.substr(0, dot);
+    }
+
+    std::string sResultXFilename =
+        sOutputDir + sFilename + "_gradientVectorPrewittBorderX_Vertical.pgm";
+    std::string sResultYFilename = sResultBaseFilename;
+
+    //        sResultXFilename += "_gradientVectorPrewittBorderX_Vertical.pgm";
+    sResultYFilename += "_gradientVectorPrewittBorderY_Horizontal.pgm";
+
+    //        if (checkCmdLineFlag(argc, (const char **)argv, "output"))
+    //        {
+    //           char *outputFilePath;
+    //            getCmdLineArgumentString(argc, (const char **)argv, "output",
+    //            &outputFilePath); sResultBaseFilename = outputFilePath;
+    //        }
+
+    // declare a host image object for an 8-bit grayscale image
+    npp::ImageCPU_8u_C1 oHostSrc;
+    // load gray-scale image from disk
+    npp::loadImage(sFilename, oHostSrc);
+    // declare a device image and copy construct from the host image,
+    // i.e. upload host to device
+    npp::ImageNPP_8u_C1 oDeviceSrc(oHostSrc);
+
+    NppiSize oSrcSize = {(int)oDeviceSrc.width(), (int)oDeviceSrc.height()};
+    NppiPoint oSrcOffset = {0, 0};
+
+    // create struct with ROI size
+    NppiSize oSizeROI = {(int)oDeviceSrc.width(), (int)oDeviceSrc.height()};
+    // allocate device destination images of appropriatedly size
+    npp::ImageNPP_16s_C1 oDeviceDstX(oSizeROI.width, oSizeROI.height);
+    npp::ImageNPP_16s_C1 oDeviceDstY(oSizeROI.width, oSizeROI.height);
+
+    // run Prewitt edge detection gradient vector filter
+    NPP_CHECK_NPP(nppiGradientVectorPrewittBorder_8u16s_C1R(
+        oDeviceSrc.data(), oDeviceSrc.pitch(), oSrcSize, oSrcOffset,
+        oDeviceDstX.data(), oDeviceDstX.pitch(), oDeviceDstY.data(),
+        oDeviceDstY.pitch(), 0, 0, 0, 0, oSizeROI, NPP_MASK_SIZE_3_X_3,
+        nppiNormL1, NPP_BORDER_REPLICATE));
+
+    // allocate device destination images of appropriatedly size
+    npp::ImageNPP_8u_C1 oDeviceDstOutX(oSizeROI.width, oSizeROI.height);
+    npp::ImageNPP_8u_C1 oDeviceDstOutY(oSizeROI.width, oSizeROI.height);
+
+    // convert 16s_C1 result images to binary 8u_C1 output images using constant
+    // value to adjust amount of visible detail
+    NPP_CHECK_NPP(nppiCompareC_16s_C1R(
+        oDeviceDstX.data(), oDeviceDstX.pitch(), 32, oDeviceDstOutX.data(),
+        oDeviceDstOutX.pitch(), oSizeROI, NPP_CMP_GREATER_EQ));
+
+    NPP_CHECK_NPP(nppiCompareC_16s_C1R(
+        oDeviceDstY.data(), oDeviceDstY.pitch(), 32, oDeviceDstOutY.data(),
+        oDeviceDstOutY.pitch(), oSizeROI, NPP_CMP_GREATER_EQ));
+
+    // create host images for the results
+    npp::ImageCPU_8u_C1 oHostDstX(oDeviceDstOutX.size());
+    npp::ImageCPU_8u_C1 oHostDstY(oDeviceDstOutY.size());
+    // and copy the device result data into them
+    oDeviceDstOutX.copyTo(oHostDstX.data(), oHostDstX.pitch());
+    oDeviceDstOutY.copyTo(oHostDstY.data(), oHostDstY.pitch());
+
+    saveImage(sResultXFilename, oHostDstX);
+    std::cout << "Saved image: " << sResultXFilename << std::endl;
+    saveImage(sResultYFilename, oHostDstY);
+    std::cout << "Saved image: " << sResultYFilename << std::endl;
+
+    // now use the Prewitt gradient border filter function in such a way that no
+    // border replication operations will be applied
+
+    // create a Prewitt filter mask size object, Prewitt uses a 3x3 filter
+    // kernel
+    NppiSize oMaskSize = {3, 3};
+    // create a size object for the enlarged source image
+    NppiSize oEnlargedSrcSize = {oSrcSize.width + oMaskSize.width - 1,
+                                 oSrcSize.height + oMaskSize.height - 1};
+
+    // create an enlarged device source image
+    npp::ImageNPP_8u_C1 oEnlargedDeviceSrc(oEnlargedSrcSize.width,
+                                           oEnlargedSrcSize.height);
+
+    // copy and enlarge the original device source image and surround it with a
+    // white edge (border)
+    NPP_CHECK_NPP(nppiCopyConstBorder_8u_C1R(
+        oDeviceSrc.data(), oDeviceSrc.pitch(), oSrcSize,
+        oEnlargedDeviceSrc.data(), oEnlargedDeviceSrc.pitch(), oEnlargedSrcSize,
+        oMaskSize.width / 2, oMaskSize.height / 2, 255));
+
+    // adjust oEnlargedDeviceSrc pixel pointer to point to the first pixel of
+    // the original source image in the enlarged source image
+    const Npp8u *pTemp =
+        reinterpret_cast<const Npp8u *>(oEnlargedDeviceSrc.data());
+    pTemp += (oMaskSize.height / 2) * oEnlargedDeviceSrc.pitch();
+    const Npp8u *pAdjustedSrc =
+        reinterpret_cast<const Npp8u *>((void *)(pTemp));
+    pAdjustedSrc += oMaskSize.width / 2;
+
+    // create device output images for the no source border results
+    npp::ImageNPP_8u_C1 oDeviceDstOutXNoBorders(oSizeROI.width,
+                                                oSizeROI.height);
+    npp::ImageNPP_8u_C1 oDeviceDstOutYNoBorders(oSizeROI.width,
+                                                oSizeROI.height);
+
+    // tell the filter function what cartesian pixel position pAdjustedSrc is
+    // pointing to within the enlarged source image
+    oSrcOffset.x += oMaskSize.width / 2;
+    oSrcOffset.y += oMaskSize.height / 2;
+
+    // run Prewitt edge detection gradient vector filter bypassing border
+    // control due to enlarged source image
+    NPP_CHECK_NPP(nppiGradientVectorPrewittBorder_8u16s_C1R(
+        pAdjustedSrc, oEnlargedDeviceSrc.pitch(), oEnlargedSrcSize, oSrcOffset,
+        oDeviceDstX.data(), oDeviceDstX.pitch(), oDeviceDstY.data(),
+        oDeviceDstY.pitch(), 0, 0, 0, 0, oSizeROI, NPP_MASK_SIZE_3_X_3,
+        nppiNormL1, NPP_BORDER_REPLICATE));
+
+    // convert 16s_C1 result images to binary 8u_C1 output images using constant
+    // value to adjust amount of visible detail
+    NPP_CHECK_NPP(nppiCompareC_16s_C1R(oDeviceDstX.data(), oDeviceDstX.pitch(),
+                                       32, oDeviceDstOutXNoBorders.data(),
+                                       oDeviceDstOutXNoBorders.pitch(),
+                                       oSizeROI, NPP_CMP_GREATER_EQ));
+
+    NPP_CHECK_NPP(nppiCompareC_16s_C1R(oDeviceDstY.data(), oDeviceDstY.pitch(),
+                                       32, oDeviceDstOutYNoBorders.data(),
+                                       oDeviceDstOutYNoBorders.pitch(),
+                                       oSizeROI, NPP_CMP_GREATER_EQ));
+    // create additional output files
+    std::string sResultXNoBordersFilename = sResultBaseFilename;
+    std::string sResultYNoBordersFilename = sResultBaseFilename;
+
+    sResultXNoBordersFilename +=
+        "_gradientVectorPrewittBorderX_Vertical_WithNoSourceBorders.pgm";
+    sResultYNoBordersFilename +=
+        "_gradientVectorPrewittBorderY_Horizontal_WithNoSourceBorders.pgm";
+
+    // copy the device result data into the host output images
+    oDeviceDstOutXNoBorders.copyTo(oHostDstX.data(), oHostDstX.pitch());
+    oDeviceDstOutYNoBorders.copyTo(oHostDstY.data(), oHostDstY.pitch());
+
+    saveImage(sResultXNoBordersFilename, oHostDstX);
+    std::cout << "Saved image: " << sResultXNoBordersFilename << std::endl;
+    saveImage(sResultYNoBordersFilename, oHostDstY);
+    std::cout << "Saved image: " << sResultYNoBordersFilename << std::endl;
+
+    // now diff the two output images, one using border control and one
+    // bypassing border control
+
+    // create device output images for the diff results
+    npp::ImageNPP_8u_C1 oDeviceDstOutXDiff(oSizeROI.width, oSizeROI.height);
+    npp::ImageNPP_8u_C1 oDeviceDstOutYDiff(oSizeROI.width, oSizeROI.height);
+
+    // diff the two 8u_C1 result images one with and one without border control
+
+    NPP_CHECK_NPP(nppiAbsDiff_8u_C1R(
+        oDeviceDstOutXNoBorders.data(), oDeviceDstOutXNoBorders.pitch(),
+        oDeviceDstOutX.data(), oDeviceDstOutX.pitch(),
+        oDeviceDstOutXDiff.data(), oDeviceDstOutXDiff.pitch(), oSizeROI));
+
+    NPP_CHECK_NPP(nppiAbsDiff_8u_C1R(
+        oDeviceDstOutYNoBorders.data(), oDeviceDstOutYNoBorders.pitch(),
+        oDeviceDstOutY.data(), oDeviceDstOutY.pitch(),
+        oDeviceDstOutYDiff.data(), oDeviceDstOutYDiff.pitch(), oSizeROI));
+
+    // create additional output files
+    std::string sResultXDiffFilename = sResultBaseFilename;
+    std::string sResultYDiffFilename = sResultBaseFilename;
+
+    sResultXDiffFilename +=
+        "_gradientVectorPrewittBorderX_Vertical_BorderDiffs.pgm";
+    sResultYDiffFilename +=
+        "_gradientVectorPrewittBorderY_Horizontal_BorderDiffs.pgm";
+
+    // copy the device result data into the host output images
+    oDeviceDstOutXDiff.copyTo(oHostDstX.data(), oHostDstX.pitch());
+    oDeviceDstOutYDiff.copyTo(oHostDstY.data(), oHostDstY.pitch());
+
+    saveImage(sResultXDiffFilename, oHostDstX);
+    std::cout << "Saved image: " << sResultXDiffFilename << std::endl;
+    saveImage(sResultYDiffFilename, oHostDstY);
+    std::cout << "Saved image: " << sResultYDiffFilename << std::endl;
+
+    // if you closely examine the above difference files (recommend using GIMP
+    // for viewing using scaling with no interpolation) you will see several
+    // single pixel differences (white pixels) along the right and bottom edges
+    // of the default vs. borderless images this happens because border pixels
+    // in the original source image are duplicated when the filter kernels
+    // overlap the edge of the source image when using the first version of the
+    // filter call but are actually sampled from the enlarged source image when
+    // using the second version of the filter call the technique used in the
+    // second filter call can be used with any filter border function in NPP to
+    // duplicate results that would be generated from a non-border filter
+    // function call by filling the border pixel outside the embedded source
+    // image with the appropriate border pixel values
+
+    // here is how to use border control to process a source image in multiple
+    // calls and get correct output in the destination image
+
+    // since the source image pointer already points to the beginning of the
+    // source image in the enlarged source image it doesn't need changed
+
+    // tighten up the top and left source image borders - this will enable
+    // border replication on the left and top borders of the original source
+    // image
+    oSrcOffset.x = 0;
+    oSrcOffset.y = 0;
+    // tighten up the right and bottom side source image borders - this will
+    // enable border replication on the right and bottom borders of the original
+    // source image
+    oEnlargedSrcSize.width = oSrcSize.width;
+    oEnlargedSrcSize.height = oSrcSize.height;
+
+    // create device output images for the mixed edge results
+    npp::ImageNPP_8u_C1 oDeviceDstOutXMixedBorders(oSizeROI.width,
+                                                   oSizeROI.height);
+    npp::ImageNPP_8u_C1 oDeviceDstOutYMixedBorders(oSizeROI.width,
+                                                   oSizeROI.height);
+
+    // shrink output ROI width so that only the left half of the destination
+    // image will be generated however since oEnlargedSrcSize.width is still set
+    // to oSrcSize.width then border control will be disabled when the filter
+    // needs to access source pixels beyond the right side of the left half of
+    // the source image
+    int nLeftWidth = oSizeROI.width / 2;
+    int nRightWidth = oSizeROI.width - nLeftWidth;
+    oSizeROI.width = nLeftWidth;
+
+    // run Prewitt edge detection gradient vector filter to generate the left
+    // side of the output image
+    NPP_CHECK_NPP(nppiGradientVectorPrewittBorder_8u16s_C1R(
+        pAdjustedSrc, oEnlargedDeviceSrc.pitch(), oEnlargedSrcSize, oSrcOffset,
+        oDeviceDstX.data(), oDeviceDstX.pitch(), oDeviceDstY.data(),
+        oDeviceDstY.pitch(), 0, 0, 0, 0, oSizeROI, NPP_MASK_SIZE_3_X_3,
+        nppiNormL1, NPP_BORDER_REPLICATE));
+
+    // now move the enlarged source pointer to the horizontal middle of the
+    // enlarged source image and tell the function where it was moved to
+    pAdjustedSrc += nLeftWidth;
+    // and adjust the source offset parameter accordingly - this will in effect
+    // turn off border control for the left border allowing the necessary source
+    // pixels to be used
+    oSrcOffset.x += nLeftWidth;
+
+    // update oSizeROI.width so that only enough destination pixels will be
+    // produced to fill the right half of the destination image
+    oSizeROI.width = nRightWidth;
+
+    // run Prewitt edge detection gradient vector filter to generate the right
+    // side of the output image adjusting the destination image pointers
+    // appropriately
+    NPP_CHECK_NPP(nppiGradientVectorPrewittBorder_8u16s_C1R(
+        pAdjustedSrc, oEnlargedDeviceSrc.pitch(), oEnlargedSrcSize, oSrcOffset,
+        oDeviceDstX.data() + nLeftWidth, oDeviceDstX.pitch(),
+        oDeviceDstY.data() + nLeftWidth, oDeviceDstY.pitch(), 0, 0, 0, 0,
+        oSizeROI, NPP_MASK_SIZE_3_X_3, nppiNormL1, NPP_BORDER_REPLICATE));
+
+    // convert 16s_C1 result images to binary 8u_C1 output images using constant
+    // value to adjust amount of visible detail
+    NPP_CHECK_NPP(nppiCompareC_16s_C1R(oDeviceDstX.data(), oDeviceDstX.pitch(),
+                                       32, oDeviceDstOutXMixedBorders.data(),
+                                       oDeviceDstOutXMixedBorders.pitch(),
+                                       oSizeROI, NPP_CMP_GREATER_EQ));
+
+    NPP_CHECK_NPP(nppiCompareC_16s_C1R(oDeviceDstY.data(), oDeviceDstY.pitch(),
+                                       32, oDeviceDstOutYMixedBorders.data(),
+                                       oDeviceDstOutYMixedBorders.pitch(),
+                                       oSizeROI, NPP_CMP_GREATER_EQ));
+    // create additional output files
+    std::string sResultXMixedBordersFilename = sResultBaseFilename;
+    std::string sResultYMixedBordersFilename = sResultBaseFilename;
+
+    sResultXMixedBordersFilename +=
+        "_gradientVectorPrewittBorderX_Vertical_WithMixedBorders.pgm";
+    sResultYMixedBordersFilename +=
+        "_gradientVectorPrewittBorderY_Horizontal_WithMixedBorders.pgm";
+
+    // copy the device result data into the host output images
+    oDeviceDstOutXMixedBorders.copyTo(oHostDstX.data(), oHostDstX.pitch());
+    oDeviceDstOutYMixedBorders.copyTo(oHostDstY.data(), oHostDstY.pitch());
+
+    saveImage(sResultXMixedBordersFilename, oHostDstX);
+    std::cout << "Saved image: " << sResultXMixedBordersFilename << std::endl;
+    saveImage(sResultYMixedBordersFilename, oHostDstY);
+    std::cout << "Saved image: " << sResultYMixedBordersFilename << std::endl;
+
+    // diff the original 8u_C1 result images with border control and the mixed
+    // border control images, they should match (diff image will be all black)
+
+    NPP_CHECK_NPP(nppiAbsDiff_8u_C1R(
+        oDeviceDstOutXMixedBorders.data(), oDeviceDstOutXMixedBorders.pitch(),
+        oDeviceDstOutX.data(), oDeviceDstOutX.pitch(),
+        oDeviceDstOutXDiff.data(), oDeviceDstOutXDiff.pitch(), oSizeROI));
+
+    NPP_CHECK_NPP(nppiAbsDiff_8u_C1R(
+        oDeviceDstOutYMixedBorders.data(), oDeviceDstOutYMixedBorders.pitch(),
+        oDeviceDstOutY.data(), oDeviceDstOutY.pitch(),
+        oDeviceDstOutYDiff.data(), oDeviceDstOutYDiff.pitch(), oSizeROI));
+
+    // create additional output files
+    std::string sResultXMixedDiffFilename = sResultBaseFilename;
+    std::string sResultYMixedDiffFilename = sResultBaseFilename;
+
+    sResultXMixedDiffFilename +=
+        "_gradientVectorPrewittBorderX_Vertical_MixedBorderDiffs.pgm";
+    sResultYMixedDiffFilename +=
+        "_gradientVectorPrewittBorderY_Horizontal_MixedBorderDiffs.pgm";
+
+    // copy the device result data into the host output images
+    oDeviceDstOutXDiff.copyTo(oHostDstX.data(), oHostDstX.pitch());
+    oDeviceDstOutYDiff.copyTo(oHostDstY.data(), oHostDstY.pitch());
+
+    saveImage(sResultXMixedDiffFilename, oHostDstX);
+    std::cout << "Saved image: " << sResultXMixedDiffFilename << std::endl;
+    saveImage(sResultYMixedDiffFilename, oHostDstY);
+    std::cout << "Saved image: " << sResultYMixedDiffFilename << std::endl;
+
+    nppiFree(oDeviceSrc.data());
+    nppiFree(oDeviceDstX.data());
+    nppiFree(oDeviceDstY.data());
+    nppiFree(oDeviceDstOutX.data());
+    nppiFree(oDeviceDstOutY.data());
+    nppiFree(oDeviceDstOutXNoBorders.data());
+    nppiFree(oDeviceDstOutYNoBorders.data());
+    nppiFree(oDeviceDstOutXDiff.data());
+    nppiFree(oDeviceDstOutYDiff.data());
+    nppiFree(oDeviceDstOutXMixedBorders.data());
+    nppiFree(oDeviceDstOutYMixedBorders.data());
+    nppiFree(oEnlargedDeviceSrc.data());
+
+    cudaDeviceReset();
+    exit(EXIT_SUCCESS);
+  } catch (npp::Exception &rException) {
+    std::cerr << "Program error! The following exception occurred: \n";
+    std::cerr << rException << std::endl;
+    std::cerr << "Aborting." << std::endl;
+
+    cudaDeviceReset();
+    exit(EXIT_FAILURE);
+  } catch (...) {
+    std::cerr << "Program error! An unknow type of exception occurred. \n";
+    std::cerr << "Aborting." << std::endl;
+
+    cudaDeviceReset();
+    exit(EXIT_FAILURE);
+    return -1;
+  }
+
+  return 0;
+}

Samples/FilterBorderControlNPP/FilterBorderControlNPP_vs2017.sln

@@ -0,0 +1,20 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio 2017
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "FilterBorderControlNPP", "FilterBorderControlNPP_vs2017.vcxproj", "{997E0757-EA74-4A4E-A0FC-47D8C8831A15}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|x64 = Debug|x64
+		Release|x64 = Release|x64
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Debug|x64.ActiveCfg = Debug|x64
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Debug|x64.Build.0 = Debug|x64
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Release|x64.ActiveCfg = Release|x64
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Release|x64.Build.0 = Release|x64
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+EndGlobal