add and update samples for CUDA 11.5

Samples/EGLStream_CUDA_CrossGPU/.vscode/c_cpp_properties.json

@@ -0,0 +1,18 @@
+{
+    "configurations": [
+        {
+            "name": "Linux",
+            "includePath": [
+                "${workspaceFolder}/**",
+                "${workspaceFolder}/../../Common"
+            ],
+            "defines": [],
+            "compilerPath": "/usr/local/cuda/bin/nvcc",
+            "cStandard": "gnu17",
+            "cppStandard": "gnu++14",
+            "intelliSenseMode": "linux-gcc-x64",
+            "configurationProvider": "ms-vscode.makefile-tools"
+        }
+    ],
+    "version": 4
+}

Samples/EGLStream_CUDA_CrossGPU/.vscode/extensions.json

@@ -0,0 +1,7 @@
+{
+    "recommendations": [
+        "nvidia.nsight-vscode-edition",
+        "ms-vscode.cpptools",
+        "ms-vscode.makefile-tools"
+    ]
+}

Samples/EGLStream_CUDA_CrossGPU/.vscode/launch.json

@@ -0,0 +1,10 @@
+{
+    "configurations": [
+        {
+            "name": "CUDA C++: Launch",
+            "type": "cuda-gdb",
+            "request": "launch",
+            "program": "${workspaceFolder}/EGLStream_CUDA_CrossGPU"
+        }
+    ]
+}

Samples/EGLStream_CUDA_CrossGPU/.vscode/tasks.json

@@ -0,0 +1,15 @@
+{
+    "version": "2.0.0",
+    "tasks": [
+        {
+            "label": "sample",
+            "type": "shell",
+            "command": "make dbg=1",
+            "problemMatcher": ["$nvcc"],
+            "group": {
+                "kind": "build",
+                "isDefault": true
+            }
+        }
+    ]
+}

Samples/EGLStream_CUDA_CrossGPU/Makefile

@@ -0,0 +1,453 @@
+################################################################################
+# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#  * Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#  * Neither the name of NVIDIA CORPORATION nor the names of its
+#    contributors may be used to endorse or promote products derived
+#    from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+################################################################################
+#
+# Makefile project only supported on Mac OS X and Linux Platforms)
+#
+################################################################################
+
+# Location of the CUDA Toolkit
+CUDA_PATH ?= /usr/local/cuda
+
+##############################
+# start deprecated interface #
+##############################
+ifeq ($(x86_64),1)
+    $(info WARNING - x86_64 variable has been deprecated)
+    $(info WARNING - please use TARGET_ARCH=x86_64 instead)
+    TARGET_ARCH ?= x86_64
+endif
+ifeq ($(ARMv7),1)
+    $(info WARNING - ARMv7 variable has been deprecated)
+    $(info WARNING - please use TARGET_ARCH=armv7l instead)
+    TARGET_ARCH ?= armv7l
+endif
+ifeq ($(aarch64),1)
+    $(info WARNING - aarch64 variable has been deprecated)
+    $(info WARNING - please use TARGET_ARCH=aarch64 instead)
+    TARGET_ARCH ?= aarch64
+endif
+ifeq ($(ppc64le),1)
+    $(info WARNING - ppc64le variable has been deprecated)
+    $(info WARNING - please use TARGET_ARCH=ppc64le instead)
+    TARGET_ARCH ?= ppc64le
+endif
+ifneq ($(GCC),)
+    $(info WARNING - GCC variable has been deprecated)
+    $(info WARNING - please use HOST_COMPILER=$(GCC) instead)
+    HOST_COMPILER ?= $(GCC)
+endif
+ifneq ($(abi),)
+    $(error ERROR - abi variable has been removed)
+endif
+############################
+# end deprecated interface #
+############################
+
+# architecture
+HOST_ARCH   := $(shell uname -m)
+TARGET_ARCH ?= $(HOST_ARCH)
+ifneq (,$(filter $(TARGET_ARCH),x86_64 aarch64 sbsa ppc64le armv7l))
+    ifneq ($(TARGET_ARCH),$(HOST_ARCH))
+        ifneq (,$(filter $(TARGET_ARCH),x86_64 aarch64 sbsa ppc64le))
+            TARGET_SIZE := 64
+        else ifneq (,$(filter $(TARGET_ARCH),armv7l))
+            TARGET_SIZE := 32
+        endif
+    else
+        TARGET_SIZE := $(shell getconf LONG_BIT)
+    endif
+else
+    $(error ERROR - unsupported value $(TARGET_ARCH) for TARGET_ARCH!)
+endif
+
+# sbsa and aarch64 systems look similar. Need to differentiate them at host level for now.
+ifeq ($(HOST_ARCH),aarch64)
+    ifeq ($(CUDA_PATH)/targets/sbsa-linux,$(shell ls -1d $(CUDA_PATH)/targets/sbsa-linux 2>/dev/null))
+        HOST_ARCH := sbsa
+        TARGET_ARCH := sbsa
+    endif
+endif
+
+ifneq ($(TARGET_ARCH),$(HOST_ARCH))
+    ifeq (,$(filter $(HOST_ARCH)-$(TARGET_ARCH),aarch64-armv7l x86_64-armv7l x86_64-aarch64 x86_64-sbsa x86_64-ppc64le))
+        $(error ERROR - cross compiling from $(HOST_ARCH) to $(TARGET_ARCH) is not supported!)
+    endif
+endif
+
+# When on native aarch64 system with userspace of 32-bit, change TARGET_ARCH to armv7l
+ifeq ($(HOST_ARCH)-$(TARGET_ARCH)-$(TARGET_SIZE),aarch64-aarch64-32)
+    TARGET_ARCH = armv7l
+endif
+
+# operating system
+HOST_OS   := $(shell uname -s 2>/dev/null | tr "[:upper:]" "[:lower:]")
+TARGET_OS ?= $(HOST_OS)
+ifeq (,$(filter $(TARGET_OS),linux darwin qnx android))
+    $(error ERROR - unsupported value $(TARGET_OS) for TARGET_OS!)
+endif
+
+# host compiler
+ifeq ($(TARGET_OS),darwin)
+    ifeq ($(shell expr `xcodebuild -version | grep -i xcode | awk '{print $$2}' | cut -d'.' -f1` \>= 5),1)
+        HOST_COMPILER ?= clang++
+    endif
+else ifneq ($(TARGET_ARCH),$(HOST_ARCH))
+    ifeq ($(HOST_ARCH)-$(TARGET_ARCH),x86_64-armv7l)
+        ifeq ($(TARGET_OS),linux)
+            HOST_COMPILER ?= arm-linux-gnueabihf-g++
+        else ifeq ($(TARGET_OS),qnx)
+            ifeq ($(QNX_HOST),)
+                $(error ERROR - QNX_HOST must be passed to the QNX host toolchain)
+            endif
+            ifeq ($(QNX_TARGET),)
+                $(error ERROR - QNX_TARGET must be passed to the QNX target toolchain)
+            endif
+            export QNX_HOST
+            export QNX_TARGET
+            HOST_COMPILER ?= $(QNX_HOST)/usr/bin/arm-unknown-nto-qnx6.6.0eabi-g++
+        else ifeq ($(TARGET_OS),android)
+            HOST_COMPILER ?= arm-linux-androideabi-g++
+        endif
+    else ifeq ($(TARGET_ARCH),aarch64)
+        ifeq ($(TARGET_OS), linux)
+            HOST_COMPILER ?= aarch64-linux-gnu-g++
+        else ifeq ($(TARGET_OS),qnx)
+            ifeq ($(QNX_HOST),)
+                $(error ERROR - QNX_HOST must be passed to the QNX host toolchain)
+            endif
+            ifeq ($(QNX_TARGET),)
+                $(error ERROR - QNX_TARGET must be passed to the QNX target toolchain)
+            endif
+            export QNX_HOST
+            export QNX_TARGET
+            HOST_COMPILER ?= $(QNX_HOST)/usr/bin/q++
+        else ifeq ($(TARGET_OS), android)
+            HOST_COMPILER ?= aarch64-linux-android-clang++
+        endif
+    else ifeq ($(TARGET_ARCH),sbsa)
+        HOST_COMPILER ?= aarch64-linux-gnu-g++
+    else ifeq ($(TARGET_ARCH),ppc64le)
+        HOST_COMPILER ?= powerpc64le-linux-gnu-g++
+    endif
+endif
+HOST_COMPILER ?= g++
+NVCC          := $(CUDA_PATH)/bin/nvcc -ccbin $(HOST_COMPILER)
+
+# internal flags
+NVCCFLAGS   := -m${TARGET_SIZE}
+CCFLAGS     :=
+LDFLAGS     :=
+
+# build flags
+ifeq ($(TARGET_OS),darwin)
+    LDFLAGS += -rpath $(CUDA_PATH)/lib
+    CCFLAGS += -arch $(HOST_ARCH)
+else ifeq ($(HOST_ARCH)-$(TARGET_ARCH)-$(TARGET_OS),x86_64-armv7l-linux)
+    LDFLAGS += --dynamic-linker=/lib/ld-linux-armhf.so.3
+    CCFLAGS += -mfloat-abi=hard
+else ifeq ($(TARGET_OS),android)
+    LDFLAGS += -pie
+    CCFLAGS += -fpie -fpic -fexceptions
+endif
+
+ifneq ($(TARGET_ARCH),$(HOST_ARCH))
+    ifeq ($(TARGET_ARCH)-$(TARGET_OS),armv7l-linux)
+        ifneq ($(TARGET_FS),)
+            GCCVERSIONLTEQ46 := $(shell expr `$(HOST_COMPILER) -dumpversion` \<= 4.6)
+            ifeq ($(GCCVERSIONLTEQ46),1)
+                CCFLAGS += --sysroot=$(TARGET_FS)
+            endif
+            LDFLAGS += --sysroot=$(TARGET_FS)
+            LDFLAGS += -rpath-link=$(TARGET_FS)/lib
+            LDFLAGS += -rpath-link=$(TARGET_FS)/usr/lib
+            LDFLAGS += -rpath-link=$(TARGET_FS)/usr/lib/arm-linux-gnueabihf
+        endif
+    endif
+    ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-linux)
+        ifneq ($(TARGET_FS),)
+            GCCVERSIONLTEQ46 := $(shell expr `$(HOST_COMPILER) -dumpversion` \<= 4.6)
+            ifeq ($(GCCVERSIONLTEQ46),1)
+                CCFLAGS += --sysroot=$(TARGET_FS)
+            endif
+            LDFLAGS += --sysroot=$(TARGET_FS)
+            LDFLAGS += -rpath-link=$(TARGET_FS)/lib -L$(TARGET_FS)/lib
+            LDFLAGS += -rpath-link=$(TARGET_FS)/lib/aarch64-linux-gnu -L$(TARGET_FS)/lib/aarch64-linux-gnu
+            LDFLAGS += -rpath-link=$(TARGET_FS)/usr/lib -L$(TARGET_FS)/usr/lib
+            LDFLAGS += -rpath-link=$(TARGET_FS)/usr/lib/aarch64-linux-gnu -L$(TARGET_FS)/usr/lib/aarch64-linux-gnu
+            LDFLAGS += --unresolved-symbols=ignore-in-shared-libs
+            CCFLAGS += -isystem=$(TARGET_FS)/usr/include -I$(TARGET_FS)/usr/include -I$(TARGET_FS)/usr/include/libdrm
+            CCFLAGS += -isystem=$(TARGET_FS)/usr/include/aarch64-linux-gnu -I$(TARGET_FS)/usr/include/aarch64-linux-gnu
+        endif
+    endif
+    ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-qnx)
+        NVCCFLAGS += --qpp-config 5.4.0,gcc_ntoaarch64le
+        CCFLAGS += -DWIN_INTERFACE_CUSTOM -I/usr/include/aarch64-qnx-gnu
+        LDFLAGS += -lsocket
+        LDFLAGS += -L/usr/lib/aarch64-qnx-gnu
+        CCFLAGS += "-Wl\,-rpath-link\,/usr/lib/aarch64-qnx-gnu"
+        ifdef TARGET_OVERRIDE
+            LDFLAGS += -lslog2
+        endif
+
+        ifneq ($(TARGET_FS),)
+            LDFLAGS += -L$(TARGET_FS)/usr/lib
+            CCFLAGS += "-Wl\,-rpath-link\,$(TARGET_FS)/usr/lib"
+            LDFLAGS += -L$(TARGET_FS)/usr/libnvidia
+            CCFLAGS += "-Wl\,-rpath-link\,$(TARGET_FS)/usr/libnvidia"
+            CCFLAGS += -I$(TARGET_FS)/../include
+        endif
+    endif
+endif
+
+ifdef TARGET_OVERRIDE # cuda toolkit targets override
+    NVCCFLAGS += -target-dir $(TARGET_OVERRIDE)
+endif
+
+# Install directory of different arch
+CUDA_INSTALL_TARGET_DIR :=
+ifeq ($(TARGET_ARCH)-$(TARGET_OS),armv7l-linux)
+    CUDA_INSTALL_TARGET_DIR = targets/armv7-linux-gnueabihf/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-linux)
+    CUDA_INSTALL_TARGET_DIR = targets/aarch64-linux/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),sbsa-linux)
+    CUDA_INSTALL_TARGET_DIR = targets/sbsa-linux/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),armv7l-android)
+    CUDA_INSTALL_TARGET_DIR = targets/armv7-linux-androideabi/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-android)
+    CUDA_INSTALL_TARGET_DIR = targets/aarch64-linux-androideabi/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),armv7l-qnx)
+    CUDA_INSTALL_TARGET_DIR = targets/ARMv7-linux-QNX/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-qnx)
+    CUDA_INSTALL_TARGET_DIR = targets/aarch64-qnx/
+else ifeq ($(TARGET_ARCH),ppc64le)
+    CUDA_INSTALL_TARGET_DIR = targets/ppc64le-linux/
+endif
+
+# Debug build flags
+ifeq ($(dbg),1)
+      NVCCFLAGS += -g -G
+      BUILD_TYPE := debug
+else
+      BUILD_TYPE := release
+endif
+
+ALL_CCFLAGS :=
+ALL_CCFLAGS += $(NVCCFLAGS)
+ALL_CCFLAGS += $(EXTRA_NVCCFLAGS)
+ALL_CCFLAGS += $(addprefix -Xcompiler ,$(CCFLAGS))
+ALL_CCFLAGS += $(addprefix -Xcompiler ,$(EXTRA_CCFLAGS))
+
+UBUNTU = $(shell lsb_release -i -s 2>/dev/null | grep -i ubuntu)
+
+SAMPLE_ENABLED := 1
+
+# This sample is not supported on Mac OSX
+ifeq ($(TARGET_OS),darwin)
+  $(info >>> WARNING - EGLStream_CUDA_CrossGPU is not supported on Mac OSX - waiving sample <<<)
+  SAMPLE_ENABLED := 0
+endif
+
+# This sample is not supported on android
+ifeq ($(TARGET_OS),android)
+  $(info >>> WARNING - EGLStream_CUDA_CrossGPU is not supported on android - waiving sample <<<)
+  SAMPLE_ENABLED := 0
+endif
+
+ALL_LDFLAGS :=
+ALL_LDFLAGS += $(ALL_CCFLAGS)
+ALL_LDFLAGS += $(addprefix -Xlinker ,$(LDFLAGS))
+ALL_LDFLAGS += $(addprefix -Xlinker ,$(EXTRA_LDFLAGS))
+
+# Common includes and paths for CUDA
+INCLUDES  := -I../../Common
+LIBRARIES :=
+
+################################################################################
+
+# Makefile include to help find EGL Libraries
+include ./findegl.mk
+
+# EGL specific libraries
+ifneq ($(TARGET_OS),darwin)
+ LIBRARIES += -lEGL
+endif
+
+#Detect if installed version of GCC supports required C++11
+ifeq ($(TARGET_OS),linux)
+    empty :=
+    space := $(empty) $(empty)
+    GCCVERSIONSTRING := $(shell expr `$(HOST_COMPILER) -dumpversion`)
+#Create version number without "."
+    GCCVERSION := $(shell expr `echo $(GCCVERSIONSTRING)` | cut -f1 -d.)
+    GCCVERSION += $(shell expr `echo $(GCCVERSIONSTRING)` | cut -f2 -d.)
+    GCCVERSION += $(shell expr `echo $(GCCVERSIONSTRING)` | cut -f3 -d.)
+# Make sure the version number has at least 3 decimals
+    GCCVERSION += 00
+# Remove spaces from the version number
+    GCCVERSION := $(subst $(space),$(empty),$(GCCVERSION))
+#$(warning $(GCCVERSION))
+
+    IS_MIN_VERSION := $(shell expr `echo $(GCCVERSION)` \>= 51000)
+
+    ifeq ($(IS_MIN_VERSION), 1)
+        $(info >>> GCC Version is greater or equal to 5.1.0 <<<)
+    else
+        $(info >>> Waiving build. Minimum GCC version required is 5.1.0<<<)
+        SAMPLE_ENABLED := 0
+    endif
+endif
+
+# Gencode arguments
+ifeq ($(TARGET_ARCH),$(filter $(TARGET_ARCH),armv7l aarch64))
+SMS ?= 35 37 50 52 60 61 70 72 75 80 86
+else
+SMS ?= 35 37 50 52 60 61 70 75 80 86
+endif
+
+ifeq ($(SMS),)
+$(info >>> WARNING - no SM architectures have been specified - waiving sample <<<)
+SAMPLE_ENABLED := 0
+endif
+
+ifeq ($(GENCODE_FLAGS),)
+# Generate SASS code for each SM architecture listed in $(SMS)
+$(foreach sm,$(SMS),$(eval GENCODE_FLAGS += -gencode arch=compute_$(sm),code=sm_$(sm)))
+
+# Generate PTX code from the highest SM architecture in $(SMS) to guarantee forward-compatibility
+HIGHEST_SM := $(lastword $(sort $(SMS)))
+ifneq ($(HIGHEST_SM),)
+GENCODE_FLAGS += -gencode arch=compute_$(HIGHEST_SM),code=compute_$(HIGHEST_SM)
+endif
+endif
+
+ifeq ($(TARGET_OS),darwin)
+  ALL_LDFLAGS += -Xcompiler -F/Library/Frameworks -Xlinker -framework -Xlinker CUDA
+else
+  ifeq ($(TARGET_ARCH),x86_64)
+    CUDA_SEARCH_PATH ?= $(CUDA_PATH)/lib64/stubs
+    CUDA_SEARCH_PATH += $(CUDA_PATH)/targets/x86_64-linux/lib/stubs
+  endif
+
+  ifeq ($(TARGET_ARCH)-$(TARGET_OS),armv7l-linux)
+    CUDA_SEARCH_PATH ?= $(CUDA_PATH)/targets/armv7-linux-gnueabihf/lib/stubs
+  endif
+
+  ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-linux)
+    CUDA_SEARCH_PATH ?= $(CUDA_PATH)/targets/aarch64-linux/lib/stubs
+  endif
+
+  ifeq ($(TARGET_ARCH)-$(TARGET_OS),sbsa-linux)
+    CUDA_SEARCH_PATH ?= $(CUDA_PATH)/targets/sbsa-linux/lib/stubs
+  endif
+
+  ifeq ($(TARGET_ARCH)-$(TARGET_OS),armv7l-android)
+    CUDA_SEARCH_PATH ?= $(CUDA_PATH)/targets/armv7-linux-androideabi/lib/stubs
+  endif
+
+  ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-android)
+    CUDA_SEARCH_PATH ?= $(CUDA_PATH)/targets/aarch64-linux-androideabi/lib/stubs
+  endif
+
+  ifeq ($(TARGET_ARCH)-$(TARGET_OS),armv7l-qnx)
+    CUDA_SEARCH_PATH ?= $(CUDA_PATH)/targets/ARMv7-linux-QNX/lib/stubs
+  endif
+
+  ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-qnx)
+    CUDA_SEARCH_PATH ?= $(CUDA_PATH)/targets/aarch64-qnx/lib/stubs
+    ifdef TARGET_OVERRIDE
+        CUDA_SEARCH_PATH := $(CUDA_PATH)/targets/$(TARGET_OVERRIDE)/lib/stubs
+    endif
+  endif
+
+  ifeq ($(TARGET_ARCH),ppc64le)
+    CUDA_SEARCH_PATH ?= $(CUDA_PATH)/targets/ppc64le-linux/lib/stubs
+  endif
+
+  ifeq ($(HOST_ARCH),ppc64le)
+    CUDA_SEARCH_PATH += $(CUDA_PATH)/lib64/stubs
+  endif
+
+  CUDALIB ?= $(shell find -L $(CUDA_SEARCH_PATH) -maxdepth 1 -name libcuda.so 2> /dev/null)
+  ifeq ("$(CUDALIB)","")
+    $(info >>> WARNING - libcuda.so not found, CUDA Driver is not installed.  Please re-install the driver. <<<)
+    SAMPLE_ENABLED := 0
+  else
+    CUDALIB := $(shell echo $(CUDALIB) | sed "s/ .*//" | sed "s/\/libcuda.so//" )
+    LIBRARIES += -L$(CUDALIB) -lcuda
+  endif
+endif
+
+ALL_CCFLAGS += --threads 0 --std=c++11
+
+ifeq ($(SAMPLE_ENABLED),0)
+EXEC ?= @echo "[@]"
+endif
+
+################################################################################
+
+# Target rules
+all: build
+
+build: EGLStream_CUDA_CrossGPU
+
+check.deps:
+ifeq ($(SAMPLE_ENABLED),0)
+	@echo "Sample will be waived due to the above missing dependencies"
+else
+	@echo "Sample is ready - all dependencies have been met"
+endif
+
+cuda_consumer.o:cuda_consumer.cpp
+	$(EXEC) $(NVCC) $(INCLUDES) $(ALL_CCFLAGS) $(GENCODE_FLAGS) -o $@ -c $<
+
+cuda_producer.o:cuda_producer.cpp
+	$(EXEC) $(NVCC) $(INCLUDES) $(ALL_CCFLAGS) $(GENCODE_FLAGS) -o $@ -c $<
+
+eglstrm_common.o:eglstrm_common.cpp
+	$(EXEC) $(NVCC) $(INCLUDES) $(ALL_CCFLAGS) $(GENCODE_FLAGS) -o $@ -c $<
+
+kernel.o:kernel.cu
+	$(EXEC) $(NVCC) $(INCLUDES) $(ALL_CCFLAGS) $(GENCODE_FLAGS) -o $@ -c $<
+
+main.o:main.cpp
+	$(EXEC) $(NVCC) $(INCLUDES) $(ALL_CCFLAGS) $(GENCODE_FLAGS) -o $@ -c $<
+
+EGLStream_CUDA_CrossGPU: cuda_consumer.o cuda_producer.o eglstrm_common.o kernel.o main.o
+	$(EXEC) $(NVCC) $(ALL_LDFLAGS) $(GENCODE_FLAGS) -o $@ $+ $(LIBRARIES)
+	$(EXEC) mkdir -p ../../bin/$(TARGET_ARCH)/$(TARGET_OS)/$(BUILD_TYPE)
+	$(EXEC) cp $@ ../../bin/$(TARGET_ARCH)/$(TARGET_OS)/$(BUILD_TYPE)
+
+run: build
+	$(EXEC) ./EGLStream_CUDA_CrossGPU
+
+clean:
+	rm -f EGLStream_CUDA_CrossGPU cuda_consumer.o cuda_producer.o eglstrm_common.o kernel.o main.o
+	rm -rf ../../bin/$(TARGET_ARCH)/$(TARGET_OS)/$(BUILD_TYPE)/EGLStream_CUDA_CrossGPU
+
+clobber: clean

Samples/EGLStream_CUDA_CrossGPU/NsightEclipse.xml

@@ -0,0 +1,78 @@
+<?xml version="1.0" encoding="UTF-8"?> 
+<!DOCTYPE entry SYSTEM "SamplesInfo.dtd">
+<entry>
+  <name>EGLStream_CUDA_CrossGPU</name>
+  <cuda_api_list>
+    <driver>cuDeviceGet</driver>
+    <driver>cuDeviceGetAttribute</driver>
+    <driver>cuDeviceComputeCapability</driver>
+    <driver>cuDeviceGetCount</driver>
+    <driver>cuDeviceGetName</driver>
+    <driver>cuGraphicsResourceGetMappedEglFrame</driver>
+    <driver>cuEGLStreamConsumerAcquireFrame</driver>
+    <driver>cuEGLStreamConsumerReleaseFrame</driver>
+    <driver>cuEGLStreamProducerReturnFrame</driver>
+    <driver>cuEGLStreamProducerPresentFrame</driver>
+    <driver>cuCtxCreate</driver>
+    <driver>cuMemAlloc</driver>
+    <driver>cuMemFree</driver>
+    <driver>cuMemcpy3D</driver>
+    <driver>cuStreamCreate</driver>
+    <driver>cuCtxPushCurrent</driver>
+    <driver>cuCtxPopCurrent</driver>
+    <driver>cuCtxDestroy</driver>
+  </cuda_api_list>
+  <description><![CDATA[Demonstrates CUDA and EGL Streams interop, where consumer's EGL Stream is on one GPU and producer's on other and both consumer-producer are different processes.]]></description>
+  <devicecompilation>whole</devicecompilation>
+  <includepaths>
+    <path>./</path>
+    <path>../</path>
+    <path>../../Common</path>
+  </includepaths>
+  <keyconcepts>
+    <concept level="basic">EGLStreams Interop</concept>
+  </keyconcepts>
+  <keywords>
+    <keyword>EGL Streams</keyword>
+  </keywords>
+  <libraries>
+    <library os="linux">cuda</library>
+    <library framework="true" os="macosx">CUDA</library>
+  </libraries>
+  <librarypaths>
+  </librarypaths>
+  <nsight_eclipse>true</nsight_eclipse>
+  <primary_file>main.cpp</primary_file>
+  <required_dependencies>
+    <dependency>EGL</dependency>
+  </required_dependencies>
+  <scopes>
+    <scope>1:CUDA Advanced Topics</scope>
+    <scope>2:Graphics Interop</scope>
+  </scopes>
+  <sm-arch>sm35</sm-arch>
+  <sm-arch>sm37</sm-arch>
+  <sm-arch>sm50</sm-arch>
+  <sm-arch>sm52</sm-arch>
+  <sm-arch>sm60</sm-arch>
+  <sm-arch>sm61</sm-arch>
+  <sm-arch>sm70</sm-arch>
+  <sm-arch>sm72</sm-arch>
+  <sm-arch>sm75</sm-arch>
+  <sm-arch>sm80</sm-arch>
+  <sm-arch>sm86</sm-arch>
+  <supported_envs>
+    <env>
+      <arch>x86_64</arch>
+      <platform>linux</platform>
+    </env>
+    <env>
+      <arch>arm</arch>
+    </env>
+  </supported_envs>
+  <supported_sm_architectures>
+    <include>all</include>
+  </supported_sm_architectures>
+  <title>EGLStream_CUDA_CrossGPU</title>
+  <type>exe</type>
+</entry>

Samples/EGLStream_CUDA_CrossGPU/README.md

@@ -0,0 +1,64 @@
+# EGLStream_CUDA_CrossGPU - EGLStream_CUDA_CrossGPU
+
+## Description
+
+Demonstrates CUDA and EGL Streams interop, where consumer's EGL Stream is on one GPU and producer's on other and both consumer-producer are different processes.
+
+## Key Concepts
+
+EGLStreams Interop
+
+## Supported SM Architectures
+
+[SM 3.5 ](https://developer.nvidia.com/cuda-gpus)  [SM 3.7 ](https://developer.nvidia.com/cuda-gpus)  [SM 5.0 ](https://developer.nvidia.com/cuda-gpus)  [SM 5.2 ](https://developer.nvidia.com/cuda-gpus)  [SM 6.0 ](https://developer.nvidia.com/cuda-gpus)  [SM 6.1 ](https://developer.nvidia.com/cuda-gpus)  [SM 7.0 ](https://developer.nvidia.com/cuda-gpus)  [SM 7.2 ](https://developer.nvidia.com/cuda-gpus)  [SM 7.5 ](https://developer.nvidia.com/cuda-gpus)  [SM 8.0 ](https://developer.nvidia.com/cuda-gpus)  [SM 8.6 ](https://developer.nvidia.com/cuda-gpus)
+
+## Supported OSes
+
+Linux
+
+## Supported CPU Architecture
+
+x86_64, armv7l
+
+## CUDA APIs involved
+
+### [CUDA Driver API](http://docs.nvidia.com/cuda/cuda-driver-api/index.html)
+cuDeviceGet, cuDeviceGetAttribute, cuDeviceComputeCapability, cuDeviceGetCount, cuDeviceGetName, cuGraphicsResourceGetMappedEglFrame, cuEGLStreamConsumerAcquireFrame, cuEGLStreamConsumerReleaseFrame, cuEGLStreamProducerReturnFrame, cuEGLStreamProducerPresentFrame, cuCtxCreate, cuMemAlloc, cuMemFree, cuMemcpy3D, cuStreamCreate, cuCtxPushCurrent, cuCtxPopCurrent, cuCtxDestroy
+
+## Dependencies needed to build/run
+[EGL](../../README.md#egl)
+
+## Prerequisites
+
+Download and install the [CUDA Toolkit 11.5](https://developer.nvidia.com/cuda-downloads) for your corresponding platform.
+Make sure the dependencies mentioned in [Dependencies]() section above are installed.
+
+## Build and Run
+
+### Linux
+The Linux samples are built using makefiles. To use the makefiles, change the current directory to the sample directory you wish to build, and run make:
+```
+$ cd <sample_dir>
+$ make
+```
+The samples makefiles can take advantage of certain options:
+*  **TARGET_ARCH=<arch>** - cross-compile targeting a specific architecture. Allowed architectures are x86_64, armv7l.
+    By default, TARGET_ARCH is set to HOST_ARCH. On a x86_64 machine, not setting TARGET_ARCH is the equivalent of setting TARGET_ARCH=x86_64.<br/>
+`$ make TARGET_ARCH=x86_64` <br/> `$ make TARGET_ARCH=armv7l` <br/>
+    See [here](http://docs.nvidia.com/cuda/cuda-samples/index.html#cross-samples) for more details.
+*   **dbg=1** - build with debug symbols
+    ```
+    $ make dbg=1
+    ```
+*   **SMS="A B ..."** - override the SM architectures for which the sample will be built, where `"A B ..."` is a space-delimited list of SM architectures. For example, to generate SASS for SM 50 and SM 60, use `SMS="50 60"`.
+    ```
+    $ make SMS="50 60"
+    ```
+
+*  **HOST_COMPILER=<host_compiler>** - override the default g++ host compiler. See the [Linux Installation Guide](http://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html#system-requirements) for a list of supported host compilers.
+```
+    $ make HOST_COMPILER=g++
+```
+
+## References (for more details)
+

Samples/EGLStream_CUDA_CrossGPU/cuda_consumer.cpp

@@ -0,0 +1,258 @@
+/* Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+//
+// DESCRIPTION:   Simple CUDA consumer rendering sample app
+//
+
+#include <cuda_runtime.h>
+#include "cuda_consumer.h"
+#include "eglstrm_common.h"
+#include <math.h>
+#include <unistd.h>
+
+#if defined(EXTENSION_LIST)
+EXTENSION_LIST(EXTLST_EXTERN)
+#endif
+CUgraphicsResource cudaResource;
+
+static int count_acq = 0;
+static double acquire_time[25000] = {0}, total_time_acq = 0;
+
+static int count_rel = 0;
+static double rel_time[25000] = {0}, total_time_rel = 0;
+
+void acquireApiStat(void);
+void acquireApiStat(void) {
+  int i = 0;
+  double min = 10000000, max = 0;
+  double average_launch_time = 0, standard_deviation = 0;
+  if (count_acq == 0) return;
+  // lets compute the standard deviation
+  min = max = acquire_time[1];
+  average_launch_time = (total_time_acq - acquire_time[0]) / count_acq;
+  for (i = 1; i < count_acq; i++) {
+    standard_deviation += (acquire_time[i] - average_launch_time) *
+                          (acquire_time[i] - average_launch_time);
+    if (acquire_time[i] < min) min = acquire_time[i];
+    if (acquire_time[i] > max) max = acquire_time[i];
+  }
+  standard_deviation = sqrt(standard_deviation / count_acq);
+  printf("acquire Avg: %lf\n", average_launch_time);
+  printf("acquire  SD: %lf\n", standard_deviation);
+  printf("acquire min: %lf\n", min);
+  printf("acquire max: %lf\n", max);
+
+  min = max = rel_time[1];
+  average_launch_time = (total_time_rel - rel_time[0]) / count_rel;
+  for (i = 1; i < count_rel; i++) {
+    standard_deviation += (rel_time[i] - average_launch_time) *
+                          (rel_time[i] - average_launch_time);
+    if (rel_time[i] < min) min = rel_time[i];
+    if (rel_time[i] > max) max = rel_time[i];
+  }
+  standard_deviation = sqrt(standard_deviation / count_rel);
+  printf("release Avg: %lf\n", average_launch_time);
+  printf("release  SD: %lf\n", standard_deviation);
+  printf("release min: %lf\n", min);
+  printf("release max: %lf\n", max);
+}
+CUresult cudaConsumerAcquireFrame(test_cuda_consumer_s *cudaConsumer,
+                                  int frameNumber) {
+  CUresult cuStatus = CUDA_SUCCESS;
+  CUeglFrame cudaEgl;
+  struct timespec start, end;
+  EGLint streamState = 0;
+  double curTime;
+
+  if (!cudaConsumer) {
+    printf("%s: Bad parameter\n", __func__);
+    goto done;
+  }
+
+  while (1) {
+    if (!eglQueryStreamKHR(cudaConsumer->eglDisplay, cudaConsumer->eglStream,
+                           EGL_STREAM_STATE_KHR, &streamState)) {
+      printf("Cuda Consumer: eglQueryStreamKHR EGL_STREAM_STATE_KHR failed\n");
+      cuStatus = CUDA_ERROR_UNKNOWN;
+      goto done;
+    }
+    if (streamState == EGL_STREAM_STATE_DISCONNECTED_KHR) {
+      printf("Cuda Consumer: EGL_STREAM_STATE_DISCONNECTED_KHR received\n");
+      cuStatus = CUDA_ERROR_UNKNOWN;
+      goto done;
+    }
+
+    if (streamState == EGL_STREAM_STATE_NEW_FRAME_AVAILABLE_KHR) {
+      break;
+    }
+  }
+  if (cudaConsumer->profileAPI) {
+    getTime(&start);
+  }
+  cuStatus =
+      cuEGLStreamConsumerAcquireFrame(&(cudaConsumer->cudaConn), &cudaResource,
+                                      &cudaConsumer->consCudaStream, 16000);
+  if (cudaConsumer->profileAPI) {
+    getTime(&end);
+    curTime = TIME_DIFF(end, start);
+    acquire_time[count_acq++] = curTime;
+    if (count_acq == 25000) count_acq = 0;
+    total_time_acq += curTime;
+  }
+  if (cuStatus == CUDA_SUCCESS) {
+    CUdeviceptr pDevPtr = 0;
+    cudaError_t err;
+
+    cuStatus =
+        cuGraphicsResourceGetMappedEglFrame(&cudaEgl, cudaResource, 0, 0);
+    if (cuStatus != CUDA_SUCCESS) {
+      printf("Cuda get resource failed with %d\n", cuStatus);
+      goto done;
+    }
+    pDevPtr = (CUdeviceptr)cudaEgl.frame.pPitch[0];
+
+    err = cudaConsumer_filter(cudaConsumer->consCudaStream, (char *)pDevPtr,
+                              WIDTH * 4, HEIGHT, PROD_DATA + frameNumber,
+                              CONS_DATA + frameNumber, frameNumber);
+    if (err != cudaSuccess) {
+      printf("Cuda Consumer: kernel failed with: %s\n",
+             cudaGetErrorString(err));
+      goto done;
+    }
+  }
+
+done:
+  return cuStatus;
+}
+
+CUresult cudaConsumerReleaseFrame(test_cuda_consumer_s *cudaConsumer,
+                                  int frameNumber) {
+  CUresult cuStatus = CUDA_SUCCESS;
+  struct timespec start, end;
+  double curTime;
+
+  if (!cudaConsumer) {
+    printf("%s: Bad parameter\n", __func__);
+    goto done;
+  }
+  if (cudaConsumer->profileAPI) {
+    getTime(&start);
+  }
+  cuStatus = cuEGLStreamConsumerReleaseFrame(
+      &cudaConsumer->cudaConn, cudaResource, &cudaConsumer->consCudaStream);
+  if (cudaConsumer->profileAPI) {
+    getTime(&end);
+    curTime = TIME_DIFF(end, start);
+    rel_time[count_rel++] = curTime;
+    if (count_rel == 25000) count_rel = 0;
+    total_time_rel += curTime;
+  }
+  if (cuStatus != CUDA_SUCCESS) {
+    printf("cuEGLStreamConsumerReleaseFrame failed, status:%d\n", cuStatus);
+    goto done;
+  }
+
+done:
+  return cuStatus;
+}
+
+CUresult cudaDeviceCreateConsumer(test_cuda_consumer_s *cudaConsumer) {
+  CUdevice device;
+  CUresult status = CUDA_SUCCESS;
+
+  if (CUDA_SUCCESS != (status = cuInit(0))) {
+    printf("Failed to initialize CUDA\n");
+    return status;
+  }
+
+  if (CUDA_SUCCESS !=
+      (status = cuDeviceGet(&device, cudaConsumer->cudaDevId))) {
+    printf("failed to get CUDA device\n");
+    return status;
+  }
+
+  if (CUDA_SUCCESS !=
+      (status = cuCtxCreate(&cudaConsumer->context, 0, device))) {
+    printf("failed to create CUDA context\n");
+    return status;
+  }
+
+  int major = 0, minor = 0;
+  char deviceName[256];
+  cuDeviceGetAttribute(&major, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR,
+                       device);
+  cuDeviceGetAttribute(&minor, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR,
+                       device);
+  cuDeviceGetName(deviceName, 256, device);
+  printf(
+      "CUDA Consumer on GPU Device %d: \"%s\" with compute capability "
+      "%d.%d\n\n",
+      device, deviceName, major, minor);
+
+  cuCtxPopCurrent(&cudaConsumer->context);
+  if (major < 6) {
+    printf(
+        "EGLStream_CUDA_CrossGPU requires SM 6.0 or higher arch GPU.  "
+        "Exiting...\n");
+    exit(2);  // EXIT_WAIVED
+  }
+
+  return status;
+}
+
+CUresult cuda_consumer_init(test_cuda_consumer_s *cudaConsumer,
+                            TestArgs *args) {
+  CUresult status = CUDA_SUCCESS;
+  int bufferSize;
+
+  cudaConsumer->charCnt = args->charCnt;
+  bufferSize = args->charCnt;
+
+  cudaConsumer->pCudaCopyMem = (unsigned char *)malloc(bufferSize);
+  if (cudaConsumer->pCudaCopyMem == NULL) {
+    printf("Cuda Consumer: malloc failed\n");
+    goto done;
+  }
+
+  status = cuStreamCreate(&cudaConsumer->consCudaStream, 0);
+  if (status != CUDA_SUCCESS) {
+    printf("Cuda Consumer: cuStreamCreate failed, status:%d\n", status);
+    goto done;
+  }
+
+  atexit(acquireApiStat);
+done:
+  return status;
+}
+
+CUresult cuda_consumer_Deinit(test_cuda_consumer_s *cudaConsumer) {
+  if (cudaConsumer->pCudaCopyMem) {
+    free(cudaConsumer->pCudaCopyMem);
+  }
+  return cuEGLStreamConsumerDisconnect(&cudaConsumer->cudaConn);
+}

Samples/EGLStream_CUDA_CrossGPU/cuda_consumer.h

@@ -0,0 +1,66 @@
+/* Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+//
+// DESCRIPTION:   CUDA consumer header file
+//
+
+#ifndef _CUDA_CONSUMER_H_
+#define _CUDA_CONSUMER_H_
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include "cudaEGL.h"
+#include "eglstrm_common.h"
+#include <cuda_runtime.h>
+#include <cuda.h>
+
+typedef struct _test_cuda_consumer_s {
+  CUcontext context;
+  CUeglStreamConnection cudaConn;
+  int cudaDevId;
+  EGLDisplay eglDisplay;
+  EGLStreamKHR eglStream;
+  unsigned int charCnt;
+  char *cudaBuf;
+  bool profileAPI;
+  unsigned char *pCudaCopyMem;
+  CUstream consCudaStream;
+} test_cuda_consumer_s;
+
+CUresult cuda_consumer_init(test_cuda_consumer_s *cudaConsumer, TestArgs *args);
+CUresult cuda_consumer_Deinit(test_cuda_consumer_s *cudaConsumer);
+CUresult cudaConsumerAcquireFrame(test_cuda_consumer_s *data, int frameNumber);
+CUresult cudaConsumerReleaseFrame(test_cuda_consumer_s *data, int frameNumber);
+CUresult cudaDeviceCreateConsumer(test_cuda_consumer_s *cudaConsumer);
+cudaError_t cudaConsumer_filter(CUstream cStream, char *pSrc, int width,
+                                int height, char expectedVal, char newVal,
+                                int frameNumber);
+cudaError_t cudaGetValueMismatch(void);
+
+#endif

Samples/EGLStream_CUDA_CrossGPU/cuda_producer.cpp

@@ -0,0 +1,288 @@
+/* Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+//
+// DESCRIPTION:   Simple cuda EGL stream producer app
+//
+
+#include "cudaEGL.h"
+#include "cuda_producer.h"
+#include "eglstrm_common.h"
+#include <cuda_runtime.h>
+#if defined(EXTENSION_LIST)
+EXTENSION_LIST(EXTLST_EXTERN)
+#endif
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <string.h>
+#include "cuda_runtime.h"
+#include "math.h"
+
+int cudaPresentReturnData = INIT_DATA;
+int fakePresent = 0;
+CUeglFrame fakeFrame;
+CUdeviceptr cudaPtrFake;
+extern bool isCrossDevice;
+
+void cudaProducerPrepareFrame(CUeglFrame *cudaEgl, CUdeviceptr cudaPtr,
+                              int bufferSize) {
+  cudaEgl->frame.pPitch[0] = (void *)cudaPtr;
+  cudaEgl->width = WIDTH;
+  cudaEgl->depth = 0;
+  cudaEgl->height = HEIGHT;
+  cudaEgl->pitch = WIDTH * 4;
+  cudaEgl->frameType = CU_EGL_FRAME_TYPE_PITCH;
+  cudaEgl->planeCount = 1;
+  cudaEgl->numChannels = 4;
+  cudaEgl->eglColorFormat = CU_EGL_COLOR_FORMAT_ARGB;
+  cudaEgl->cuFormat = CU_AD_FORMAT_UNSIGNED_INT8;
+}
+
+static int count_present = 0, count_return = 0;
+static double present_time[25000] = {0}, total_time_present = 0;
+static double return_time[25000] = {0}, total_time_return = 0;
+
+void presentApiStat(void);
+void presentApiStat(void) {
+  int i = 0;
+  double min = 10000000, max = 0;
+  double average_launch_time = 0, standard_deviation = 0;
+  if (count_present == 0) return;
+  // lets compute the standard deviation
+  min = max = present_time[1];
+  average_launch_time = (total_time_present) / count_present;
+  for (i = 1; i < count_present; i++) {
+    standard_deviation += (present_time[i] - average_launch_time) *
+                          (present_time[i] - average_launch_time);
+    if (present_time[i] < min) min = present_time[i];
+    if (present_time[i] > max) max = present_time[i];
+  }
+  standard_deviation = sqrt(standard_deviation / count_present);
+  printf("present Avg: %lf\n", average_launch_time);
+  printf("present  SD: %lf\n", standard_deviation);
+  printf("present min: %lf\n", min);
+  printf("present max: %lf\n", max);
+
+  min = max = return_time[1];
+  average_launch_time = (total_time_return - return_time[0]) / count_return;
+  for (i = 1; i < count_return; i++) {
+    standard_deviation += (return_time[i] - average_launch_time) *
+                          (return_time[i] - average_launch_time);
+    if (return_time[i] < min) min = return_time[i];
+    if (return_time[i] > max) max = return_time[i];
+  }
+  standard_deviation = sqrt(standard_deviation / count_return);
+  printf("return  Avg: %lf\n", average_launch_time);
+  printf("return   SD: %lf\n", standard_deviation);
+  printf("return  min: %lf\n", min);
+  printf("return  max: %lf\n", max);
+}
+CUresult cudaProducerPresentFrame(test_cuda_producer_s *cudaProducer,
+                                  CUeglFrame cudaEgl, int t) {
+  static int flag = 0;
+  CUresult status = CUDA_SUCCESS;
+  struct timespec start, end;
+  double curTime;
+  CUdeviceptr pDevPtr = (CUdeviceptr)cudaEgl.frame.pPitch[0];
+  cudaProducer_filter(cudaProducer->prodCudaStream, (char *)pDevPtr, WIDTH * 4,
+                      HEIGHT, cudaPresentReturnData, PROD_DATA + t, t);
+  if (cudaProducer->profileAPI) {
+    getTime(&start);
+  }
+  status = cuEGLStreamProducerPresentFrame(&cudaProducer->cudaConn, cudaEgl,
+                                           &cudaProducer->prodCudaStream);
+  if (status != CUDA_SUCCESS) {
+    printf("Cuda Producer: Present frame failed, status:%d\n", status);
+    goto done;
+  }
+  flag++;
+  if (cudaProducer->profileAPI && flag > 10) {
+    getTime(&end);
+    curTime = TIME_DIFF(end, start);
+    present_time[count_present++] = curTime;
+    if (count_present == 25000) count_present = 0;
+    total_time_present += curTime;
+  }
+done:
+  return status;
+}
+
+int flag = 0;
+CUresult cudaProducerReturnFrame(test_cuda_producer_s *cudaProducer,
+                                 CUeglFrame cudaEgl, int t) {
+  CUresult status = CUDA_SUCCESS;
+  struct timespec start, end;
+  double curTime;
+  CUdeviceptr pDevPtr = 0;
+
+  pDevPtr = (CUdeviceptr)cudaEgl.frame.pPitch[0];
+  if (cudaProducer->profileAPI) {
+    getTime(&start);
+  }
+
+  while (1) {
+    status = cuEGLStreamProducerReturnFrame(&cudaProducer->cudaConn, &cudaEgl,
+                                            &cudaProducer->prodCudaStream);
+    if (status == CUDA_ERROR_LAUNCH_TIMEOUT) {
+      continue;
+    } else if (status != CUDA_SUCCESS) {
+      printf("Cuda Producer: Return frame failed, status:%d\n", status);
+      goto done;
+    }
+    break;
+  }
+  if (cudaProducer->profileAPI) {
+    getTime(&end);
+    curTime = TIME_DIFF(end, start);
+    return_time[count_return++] = curTime;
+    if (count_return == 25000) count_return = 0;
+    total_time_return += curTime;
+  }
+  if (flag % 2 == 0) {
+    cudaPresentReturnData++;
+  }
+  cudaProducer_filter(cudaProducer->prodCudaStream, (char *)pDevPtr, WIDTH * 4,
+                      HEIGHT, CONS_DATA + t, cudaPresentReturnData, t);
+  flag++;
+done:
+  return status;
+}
+
+CUresult cudaDeviceCreateProducer(test_cuda_producer_s *cudaProducer) {
+  CUdevice device;
+  CUresult status = CUDA_SUCCESS;
+
+  if (CUDA_SUCCESS != (status = cuInit(0))) {
+    printf("Failed to initialize CUDA\n");
+    return status;
+  }
+
+  if (CUDA_SUCCESS !=
+      (status = cuDeviceGet(&device, cudaProducer->cudaDevId))) {
+    printf("failed to get CUDA device\n");
+    return status;
+  }
+
+  if (CUDA_SUCCESS !=
+      (status = cuCtxCreate(&cudaProducer->context, 0, device))) {
+    printf("failed to create CUDA context\n");
+    return status;
+  }
+
+  int major = 0, minor = 0;
+  char deviceName[256];
+  cuDeviceGetAttribute(&major, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR,
+                       device);
+  cuDeviceGetAttribute(&minor, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR,
+                       device);
+  cuDeviceGetName(deviceName, 256, device);
+  printf(
+      "CUDA Producer on GPU Device %d: \"%s\" with compute capability "
+      "%d.%d\n\n",
+      device, deviceName, major, minor);
+
+  cuCtxPopCurrent(&cudaProducer->context);
+
+  if (major < 6) {
+    printf(
+        "EGLStream_CUDA_CrossGPU requires SM 6.0 or higher arch GPU.  "
+        "Exiting...\n");
+    exit(2);  // EXIT_WAIVED
+  }
+
+  return status;
+}
+
+CUresult cudaProducerInit(test_cuda_producer_s *cudaProducer, TestArgs *args) {
+  CUresult status = CUDA_SUCCESS;
+  int bufferSize;
+
+  cudaProducer->charCnt = args->charCnt;
+  bufferSize = cudaProducer->charCnt;
+
+  cudaProducer->tempBuff = (char *)malloc(bufferSize);
+  if (!cudaProducer->tempBuff) {
+    printf("Cuda Producer: Failed to allocate image buffer\n");
+    status = CUDA_ERROR_UNKNOWN;
+    goto done;
+  }
+  memset((void *)cudaProducer->tempBuff, INIT_DATA, cudaProducer->charCnt);
+
+  // Fill this init data
+  status = cuMemAlloc(&cudaProducer->cudaPtr, bufferSize);
+  if (status != CUDA_SUCCESS) {
+    printf("Cuda Producer: cuda Malloc failed, status:%d\n", status);
+    goto done;
+  }
+  status = cuMemcpyHtoD(cudaProducer->cudaPtr, (void *)(cudaProducer->tempBuff),
+                        bufferSize);
+  if (status != CUDA_SUCCESS) {
+    printf("Cuda Producer: cuMemCpy failed, status:%d\n", status);
+    goto done;
+  }
+
+  // Fill this init data
+  status = cuMemAlloc(&cudaProducer->cudaPtr1, bufferSize);
+  if (status != CUDA_SUCCESS) {
+    printf("Cuda Producer: cuda Malloc failed, status:%d\n", status);
+    goto done;
+  }
+  status = cuMemcpyHtoD(cudaProducer->cudaPtr1,
+                        (void *)(cudaProducer->tempBuff), bufferSize);
+  if (status != CUDA_SUCCESS) {
+    printf("Cuda Producer: cuMemCpy failed, status:%d\n", status);
+    goto done;
+  }
+
+  status = cuStreamCreate(&cudaProducer->prodCudaStream, 0);
+  if (status != CUDA_SUCCESS) {
+    printf("Cuda Producer: cuStreamCreate failed, status:%d\n", status);
+    goto done;
+  }
+
+  // Fill this init data
+  status = cuMemAlloc(&cudaPtrFake, 100);
+  if (status != CUDA_SUCCESS) {
+    printf("Cuda Producer: cuda Malloc failed, status:%d\n", status);
+    goto done;
+  }
+
+  atexit(presentApiStat);
+done:
+  return status;
+}
+
+CUresult cudaProducerDeinit(test_cuda_producer_s *cudaProducer) {
+  if (cudaProducer->tempBuff) {
+    free(cudaProducer->tempBuff);
+  }
+  if (cudaProducer->cudaPtr) {
+    cuMemFree(cudaProducer->cudaPtr);
+  }
+  return cuEGLStreamProducerDisconnect(&cudaProducer->cudaConn);
+}

Samples/EGLStream_CUDA_CrossGPU/cuda_producer.h

@@ -0,0 +1,68 @@
+/* Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+//
+// DESCRIPTION:   Simple cuda producer header file
+//
+
+#ifndef _CUDA_PRODUCER_H_
+#define _CUDA_PRODUCER_H_
+#include <EGL/egl.h>
+#include <EGL/eglext.h>
+#include "cudaEGL.h"
+#include "eglstrm_common.h"
+#include <cuda_runtime.h>
+#include <cuda.h>
+
+typedef struct _test_cuda_producer_s {
+  //  Stream params
+  CUcontext context;
+  CUeglStreamConnection cudaConn;
+  int cudaDevId;
+  EGLStreamKHR eglStream;
+  EGLDisplay eglDisplay;
+  unsigned int charCnt;
+  bool profileAPI;
+  char *tempBuff;
+  CUdeviceptr cudaPtr;
+  CUdeviceptr cudaPtr1;
+  CUstream prodCudaStream;
+} test_cuda_producer_s;
+
+CUresult cudaProducerInit(test_cuda_producer_s *cudaProducer, TestArgs *args);
+CUresult cudaProducerPresentFrame(test_cuda_producer_s *parserArg,
+                                  CUeglFrame cudaEgl, int t);
+CUresult cudaProducerReturnFrame(test_cuda_producer_s *parserArg,
+                                 CUeglFrame cudaEgl, int t);
+CUresult cudaProducerDeinit(test_cuda_producer_s *cudaProducer);
+CUresult cudaDeviceCreateProducer(test_cuda_producer_s *cudaProducer);
+cudaError_t cudaProducer_filter(CUstream cStream, char *pSrc, int width,
+                                int height, char expectedVal, char newVal,
+                                int frameNumber);
+void cudaProducerPrepareFrame(CUeglFrame *cudaEgl, CUdeviceptr cudaPtr,
+                              int bufferSize);
+#endif

Samples/EGLStream_CUDA_CrossGPU/eglstrm_common.cpp

@@ -0,0 +1,423 @@
+/* Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+//
+// DESCRIPTION:   Common egl stream functions
+//
+
+#include "eglstrm_common.h"
+
+EGLStreamKHR g_producerEglStream = EGL_NO_STREAM_KHR;
+EGLStreamKHR g_consumerEglStream = EGL_NO_STREAM_KHR;
+EGLDisplay g_producerEglDisplay = EGL_NO_DISPLAY;
+EGLDisplay g_consumerEglDisplay = EGL_NO_DISPLAY;
+int cudaDevIndexProd = -1;
+int cudaDevIndexCons = -1;
+
+#if defined(EXTENSION_LIST)
+EXTENSION_LIST(EXTLST_DECL)
+typedef void (*extlst_fnptr_t)(void);
+static struct {
+  extlst_fnptr_t *fnptr;
+  char const *name;
+  bool is_dgpu;  // This function is need only for dgpu case
+} extensionList[] = {EXTENSION_LIST(EXTLST_ENTRY)};
+
+int eglSetupExtensions(bool isCrossDevice) {
+  unsigned int i;
+
+  for (i = 0; i < (sizeof(extensionList) / sizeof(*extensionList)); i++) {
+    // load the dgpu function only if we are running cross device test
+    if ((!extensionList[i].is_dgpu) ||
+        (extensionList[i].is_dgpu == isCrossDevice)) {
+      *extensionList[i].fnptr = eglGetProcAddress(extensionList[i].name);
+      if (*extensionList[i].fnptr == NULL) {
+        printf("Couldn't get address of %s()\n", extensionList[i].name);
+        return 0;
+      }
+    }
+  }
+
+  return 1;
+}
+
+int EGLStreamInit(bool isCrossDevice, int isConsumer,
+                  EGLNativeFileDescriptorKHR fileDesc) {
+  static const EGLint streamAttrFIFOMode[] = {
+      EGL_STREAM_FIFO_LENGTH_KHR, 5, EGL_SUPPORT_REUSE_NV, EGL_FALSE, EGL_NONE};
+  EGLDisplay eglDisplay[2] = {0};
+  EGLStreamKHR eglStream[2] = {0};
+  EGLBoolean eglStatus;
+
+#define MAX_EGL_DEVICES 4
+
+  EGLDeviceEXT devices[MAX_EGL_DEVICES];
+  EGLint numDevices = 0;
+
+  eglStatus = eglQueryDevicesEXT(MAX_EGL_DEVICES, devices, &numDevices);
+  if (eglStatus != EGL_TRUE) {
+    printf("Error querying EGL devices\n");
+    goto Done;
+  }
+
+  if (numDevices == 0) {
+    printf("No EGL devices found\n");
+    eglStatus = EGL_FALSE;
+    goto Done;
+  }
+
+  // If cross device, create discrete GPU stream first and then create the
+  // integrated GPU stream to connect to it via fd. The other way round fails
+  // in producer connect.
+  //
+  // TODO: Find out if this EGL behavior is by design.
+  if (isConsumer) {
+    int egl_device_id = 0;
+    for (egl_device_id = 0; egl_device_id < numDevices; egl_device_id++) {
+      EGLAttrib cuda_device;
+      eglStatus = eglQueryDeviceAttribEXT(devices[egl_device_id],
+                                          EGL_CUDA_DEVICE_NV, &cuda_device);
+      if (eglStatus == EGL_TRUE) {
+        cudaDevIndexCons = cuda_device;  // We select first EGL-CUDA Capable
+                                         // device for consumer.
+        printf(
+            "Found EGL-CUDA Capable device with CUDA Device id = %d out of "
+            "egl_device_id = %d\n",
+            (int)cudaDevIndexCons, egl_device_id);
+        break;
+      }
+    }
+
+    if (egl_device_id >= numDevices) {
+      printf("No CUDA Capable EGL Device found.. Waiving execution\n");
+      goto Done;
+    }
+
+    g_consumerEglDisplay = eglGetPlatformDisplayEXT(
+        EGL_PLATFORM_DEVICE_EXT, (void *)devices[egl_device_id], NULL);
+    if (g_consumerEglDisplay == EGL_NO_DISPLAY) {
+      printf("Could not get EGL display from device. \n");
+      eglStatus = EGL_FALSE;
+      goto Done;
+    }
+
+    eglStatus = eglInitialize(g_consumerEglDisplay, 0, 0);
+    if (!eglStatus) {
+      printf("EGL failed to initialize. \n");
+      eglStatus = EGL_FALSE;
+      goto Done;
+    }
+
+    g_consumerEglStream =
+        eglCreateStreamKHR(g_consumerEglDisplay, streamAttrFIFOMode);
+    if (g_consumerEglStream == EGL_NO_STREAM_KHR) {
+      printf("Could not create EGL stream.\n");
+      eglStatus = EGL_FALSE;
+      goto Done;
+    }
+
+    eglStatus = eglStreamAttribKHR(g_consumerEglDisplay, g_consumerEglStream,
+                                   EGL_CONSUMER_LATENCY_USEC_KHR, 16000);
+    if (eglStatus != EGL_TRUE) {
+      printf("eglStreamAttribKHR EGL_CONSUMER_LATENCY_USEC_KHR failed\n");
+      goto Done;
+    }
+
+    eglStatus =
+        eglStreamAttribKHR(g_consumerEglDisplay, g_consumerEglStream,
+                           EGL_CONSUMER_ACQUIRE_TIMEOUT_USEC_KHR, 16000);
+    if (eglStatus != EGL_TRUE) {
+      printf(
+          "eglStreamAttribKHR EGL_CONSUMER_ACQUIRE_TIMEOUT_USEC_KHR failed\n");
+      goto Done;
+    }
+  }
+
+  if (!isConsumer) {  // Producer
+
+    if (fileDesc == EGL_NO_FILE_DESCRIPTOR_KHR) {
+      printf("Cuda Producer received bad file descriptor\n");
+      eglStatus = EGL_FALSE;
+      goto Done;
+    }
+
+    int egl_device_id = 0;
+    int egl_cuda_devices = 0;
+    for (egl_device_id = 0; egl_device_id < numDevices; egl_device_id++) {
+      EGLAttrib cuda_device = -1;
+      eglStatus = eglQueryDeviceAttribEXT(devices[egl_device_id],
+                                          EGL_CUDA_DEVICE_NV, &cuda_device);
+      if (eglStatus == EGL_TRUE) {
+        egl_cuda_devices++;
+        if (isCrossDevice && (egl_cuda_devices > 1)) {
+          // We select second EGL-CUDA Capable device for producer.
+          cudaDevIndexProd = (int)cuda_device;
+          printf(
+              "Found EGL-CUDA Capable device with CUDA Device id = %d "
+              "egl_device_id = %d \n",
+              (int)cudaDevIndexProd, egl_device_id);
+          break;
+        }
+        if (!isCrossDevice) {
+          // We select first EGL-CUDA Capable device for producer same as
+          // consumer.
+          cudaDevIndexProd = (int)cuda_device;
+          printf(
+              "Found EGL-CUDA Capable device with CUDA Device id = %d "
+              "egl_device_id = %d \n",
+              (int)cudaDevIndexProd, egl_device_id);
+          break;
+        }
+      }
+    }
+
+    if (egl_device_id >= numDevices) {
+      printf("No CUDA Capable EGL Device found.. Waiving execution\n");
+      goto Done;
+    }
+
+    g_producerEglDisplay = eglGetPlatformDisplayEXT(
+        EGL_PLATFORM_DEVICE_EXT, (void *)devices[egl_device_id], NULL);
+    if (g_producerEglDisplay == EGL_NO_DISPLAY) {
+      printf("Could not get EGL display from device. \n");
+      eglStatus = EGL_FALSE;
+      goto Done;
+    }
+
+    eglStatus = eglInitialize(g_producerEglDisplay, 0, 0);
+    if (!eglStatus) {
+      printf("EGL failed to initialize. \n");
+      eglStatus = EGL_FALSE;
+      goto Done;
+    }
+
+    g_producerEglStream =
+        eglCreateStreamFromFileDescriptorKHR(g_producerEglDisplay, fileDesc);
+    close(fileDesc);
+
+    if (g_producerEglStream == EGL_NO_STREAM_KHR) {
+      printf("CUDA Producer Could not create EGL stream.\n");
+      eglStatus = EGL_FALSE;
+      goto Done;
+    } else {
+      printf("Producer created EGLStream for the GPU.\n");
+    }
+  }
+
+Done:
+  return eglStatus == EGL_TRUE ? 1 : 0;
+}
+
+void EGLStreamFini(void) {
+  if (g_producerEglStream != EGL_NO_STREAM_KHR) {
+    eglDestroyStreamKHR(g_producerEglDisplay, g_producerEglStream);
+  }
+  if (g_consumerEglStream != g_producerEglStream) {
+    if (g_consumerEglStream != EGL_NO_STREAM_KHR) {
+      eglDestroyStreamKHR(g_consumerEglDisplay, g_consumerEglStream);
+    }
+  }
+}
+
+int UnixSocketConnect(const char *socket_name) {
+  int sock_fd = -1;
+  struct sockaddr_un sock_addr;
+  int wait_loop = 0;
+
+  sock_fd = socket(PF_UNIX, SOCK_STREAM, 0);
+  if (sock_fd < 0) {
+    printf("%s: socket create failed.\n", __func__);
+    return -1;
+  }
+
+  if (verbose) printf("%s: send_fd: sock_fd: %d\n", __func__, sock_fd);
+
+  memset(&sock_addr, 0, sizeof(struct sockaddr_un));
+  sock_addr.sun_family = AF_UNIX;
+  strncpy(sock_addr.sun_path, socket_name, sizeof(sock_addr.sun_path) - 1);
+
+  while (connect(sock_fd, (const struct sockaddr *)&sock_addr,
+                 sizeof(struct sockaddr_un))) {
+    if (wait_loop < 60) {
+      if (!wait_loop)
+        printf("Waiting for EGL stream producer ");
+      else
+        printf(".");
+      fflush(stdout);
+      sleep(1);
+      wait_loop++;
+    } else {
+      printf("\n%s: Waiting timed out\n", __func__);
+      return -1;
+    }
+  }
+  if (wait_loop) printf("\n");
+
+  if (verbose) printf("%s: Wait is done\n", __func__);
+
+  return sock_fd;
+}
+
+/* Send <fd_to_send> (a file descriptor) to another process */
+/* over a unix domain socket named <socket_name>.           */
+/* <socket_name> can be any nonexistant filename.           */
+int EGLStreamSendfd(int send_fd, int fd_to_send) {
+  struct msghdr msg;
+  struct iovec iov[1];
+  char ctrl_buf[CMSG_SPACE(sizeof(int))];
+  struct cmsghdr *cmsg = NULL;
+  void *data;
+  int res;
+  memset(&msg, 0, sizeof(msg));
+
+  iov[0].iov_len = 1;             // must send at least 1 byte
+  iov[0].iov_base = (void *)"x";  // any byte value (value ignored)
+  msg.msg_iov = iov;
+  msg.msg_iovlen = 1;
+
+  memset(ctrl_buf, 0, sizeof(ctrl_buf));
+  msg.msg_control = ctrl_buf;
+  msg.msg_controllen = sizeof(ctrl_buf);
+
+  cmsg = CMSG_FIRSTHDR(&msg);
+  cmsg->cmsg_level = SOL_SOCKET;
+  cmsg->cmsg_type = SCM_RIGHTS;
+  cmsg->cmsg_len = CMSG_LEN(sizeof(int));
+  data = CMSG_DATA(cmsg);
+  *(int *)data = fd_to_send;
+
+  msg.msg_controllen = cmsg->cmsg_len;
+
+  res = sendmsg(send_fd, &msg, 0);
+  if (res <= 0) {
+    printf("%s: sendmsg failed", __func__);
+    return -1;
+  }
+
+  return 0;
+}
+
+/* Listen on a unix domain socket named <socket_name>.     */
+/* Connect to it and return connect_fd                     */
+int UnixSocketCreate(const char *socket_name) {
+  int listen_fd;
+  struct sockaddr_un sock_addr;
+  int connect_fd;
+  struct sockaddr_un connect_addr;
+  socklen_t connect_addr_len = 0;
+
+  listen_fd = socket(PF_UNIX, SOCK_STREAM, 0);
+  if (listen_fd < 0) {
+    printf("%s: socket create failed", __func__);
+    return -1;
+  }
+
+  if (verbose) printf("%s: listen_fd: %d\n", __func__, listen_fd);
+
+  unlink(socket_name);
+
+  memset(&sock_addr, 0, sizeof(struct sockaddr_un));
+  sock_addr.sun_family = AF_UNIX;
+  strncpy(sock_addr.sun_path, socket_name, sizeof(sock_addr.sun_path) - 1);
+
+  if (bind(listen_fd, (const struct sockaddr *)&sock_addr,
+           sizeof(struct sockaddr_un))) {
+    printf("i%s: bind error", __func__);
+    return -1;
+  }
+
+  if (listen(listen_fd, 1)) {
+    printf("%s: listen error", __func__);
+    return -1;
+  }
+
+  connect_fd =
+      accept(listen_fd, (struct sockaddr *)&connect_addr, &connect_addr_len);
+
+  if (verbose) printf("%s: connect_fd: %d\n", __func__, connect_fd);
+
+  close(listen_fd);
+  unlink(socket_name);
+  if (connect_fd < 0) {
+    printf("%s: accept failed\n", __func__);
+    return -1;
+  }
+
+  return connect_fd;
+}
+
+/* receive a file descriptor from another process.         */
+/* Returns the file descriptor.  Note: the integer value   */
+/* of the file descriptor may be different from the        */
+/* integer value in the other process, but the file        */
+/* descriptors in each process will refer to the same file */
+/* object in the kernel.                                   */
+int EGLStreamReceivefd(int connect_fd) {
+  struct msghdr msg;
+  struct iovec iov[1];
+  char msg_buf[1];
+  char ctrl_buf[CMSG_SPACE(sizeof(int))];
+  struct cmsghdr *cmsg;
+  void *data;
+  int recvfd;
+
+  memset(&msg, 0, sizeof(msg));
+
+  iov[0].iov_base = msg_buf;
+  iov[0].iov_len = sizeof(msg_buf);
+  msg.msg_iov = iov;
+  msg.msg_iovlen = 1;
+
+  msg.msg_control = ctrl_buf;
+  msg.msg_controllen = sizeof(ctrl_buf);
+
+  if (recvmsg(connect_fd, &msg, 0) <= 0) {
+    printf("%s: recvmsg failed", __func__);
+    return -1;
+  }
+
+  cmsg = CMSG_FIRSTHDR(&msg);
+  if (!cmsg) {
+    printf("%s: NULL message header\n", __func__);
+    return -1;
+  }
+  if (cmsg->cmsg_level != SOL_SOCKET) {
+    printf("%s: Message level is not SOL_SOCKET\n", __func__);
+    return -1;
+  }
+  if (cmsg->cmsg_type != SCM_RIGHTS) {
+    printf("%s: Message type is not SCM_RIGHTS\n", __func__);
+    return -1;
+  }
+
+  data = CMSG_DATA(cmsg);
+  recvfd = *(int *)data;
+
+  return recvfd;
+}
+
+#endif

Samples/EGLStream_CUDA_CrossGPU/eglstrm_common.h

@@ -0,0 +1,113 @@
+/* Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+//
+// DESCRIPTION:   Common EGL stream functions header file
+//
+
+#ifndef _EGLSTRM_COMMON_H_
+#define _EGLSTRM_COMMON_H_
+
+#include <signal.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/socket.h>
+#include <sys/stat.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <sys/un.h>
+#include <time.h>
+#include <unistd.h>
+
+#include "cuda.h"
+#include "cudaEGL.h"
+#define TIME_DIFF(end, start) (getMicrosecond(end) - getMicrosecond(start))
+
+extern EGLStreamKHR g_producerEglStream;
+extern EGLStreamKHR g_consumerEglStream;
+extern EGLDisplay g_producerEglDisplay;
+extern EGLDisplay g_consumerEglDisplay;
+extern int cudaDevIndexCons;
+extern int cudaDevIndexProd;
+extern bool verbose;
+
+#define EXTENSION_LIST(T)                                                \
+  T(PFNEGLCREATESTREAMKHRPROC, eglCreateStreamKHR)                       \
+  T(PFNEGLDESTROYSTREAMKHRPROC, eglDestroyStreamKHR)                     \
+  T(PFNEGLQUERYSTREAMKHRPROC, eglQueryStreamKHR)                         \
+  T(PFNEGLQUERYSTREAMU64KHRPROC, eglQueryStreamu64KHR)                   \
+  T(PFNEGLQUERYSTREAMTIMEKHRPROC, eglQueryStreamTimeKHR)                 \
+  T(PFNEGLSTREAMATTRIBKHRPROC, eglStreamAttribKHR)                       \
+  T(PFNEGLSTREAMCONSUMERACQUIREKHRPROC, eglStreamConsumerAcquireKHR)     \
+  T(PFNEGLSTREAMCONSUMERRELEASEKHRPROC, eglStreamConsumerReleaseKHR)     \
+  T(PFNEGLSTREAMCONSUMERGLTEXTUREEXTERNALKHRPROC,                        \
+    eglStreamConsumerGLTextureExternalKHR)                               \
+  T(PFNEGLQUERYDEVICESEXTPROC, eglQueryDevicesEXT)                       \
+  T(PFNEGLGETPLATFORMDISPLAYEXTPROC, eglGetPlatformDisplayEXT)           \
+  T(PFNEGLGETSTREAMFILEDESCRIPTORKHRPROC, eglGetStreamFileDescriptorKHR) \
+  T(PFNEGLQUERYDEVICEATTRIBEXTPROC, eglQueryDeviceAttribEXT)             \
+  T(PFNEGLCREATESTREAMFROMFILEDESCRIPTORKHRPROC,                         \
+    eglCreateStreamFromFileDescriptorKHR)
+
+#define EXTLST_DECL(tx, x) tx x = NULL;
+#define EXTLST_EXTERN(tx, x) extern tx x;
+#define EXTLST_ENTRY(tx, x) {(extlst_fnptr_t *)&x, #x},
+
+#define MAX_STRING_SIZE 256
+#define INIT_DATA 0x01
+#define PROD_DATA 0x07
+#define CONS_DATA 0x04
+
+#define SOCK_PATH "/tmp/tegra_sw_egl_socket"
+
+typedef struct _TestArgs {
+  unsigned int charCnt;
+  bool isProducer;
+} TestArgs;
+
+extern int WIDTH, HEIGHT;
+
+int eglSetupExtensions(bool is_dgpu);
+int EGLStreamInit(bool isCrossDevice, int isConsumer,
+                  EGLNativeFileDescriptorKHR fileDesc);
+void EGLStreamFini(void);
+
+int EGLStreamSetAttr(EGLDisplay display, EGLStreamKHR eglStream);
+int UnixSocketConnect(const char *socket_name);
+int EGLStreamSendfd(int send_fd, int fd_to_send);
+int UnixSocketCreate(const char *socket_name);
+int EGLStreamReceivefd(int connect_fd);
+
+static clockid_t clock_id = CLOCK_MONOTONIC;  // CLOCK_PROCESS_CPUTIME_ID;
+static double getMicrosecond(struct timespec t) {
+  return ((t.tv_sec) * 1000000.0 + (t.tv_nsec) / 1.0e3);
+}
+
+static inline void getTime(struct timespec *t) { clock_gettime(clock_id, t); }
+#endif

Samples/EGLStream_CUDA_CrossGPU/findegl.mk

@@ -0,0 +1,160 @@
+################################################################################
+# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#  * Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#  * Neither the name of NVIDIA CORPORATION nor the names of its
+#    contributors may be used to endorse or promote products derived
+#    from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+################################################################################
+#
+#  findegl.mk is used to find the necessary EGL Libraries for specific distributions
+#               this is supported on Linux
+#
+################################################################################
+
+# Determine OS platform and unix distribution
+ifeq ("$(TARGET_OS)","linux")
+   # first search lsb_release
+   DISTRO  = $(shell lsb_release -i -s 2>/dev/null | tr "[:upper:]" "[:lower:]")
+   ifeq ("$(DISTRO)","")
+     # second search and parse /etc/issue
+     DISTRO = $(shell more /etc/issue | awk '{print $$1}' | sed '1!d' | sed -e "/^$$/d" 2>/dev/null | tr "[:upper:]" "[:lower:]")
+     # ensure data from /etc/issue is valid
+     ifeq (,$(filter $(DISTRO),ubuntu fedora red rhel centos suse))
+       DISTRO = 
+     endif
+     ifeq ("$(DISTRO)","")
+       # third, we can search in /etc/os-release or /etc/{distro}-release
+       DISTRO = $(shell awk '/ID/' /etc/*-release | sed 's/ID=//' | grep -v "VERSION" | grep -v "ID" | grep -v "DISTRIB")
+     endif
+   endif
+endif
+
+ifeq ("$(TARGET_OS)","linux")
+    # $(info) >> findegl.mk -> LINUX path <<<)
+    # Each set of Linux Distros have different paths for where to find their OpenGL libraries reside
+    UBUNTU = $(shell echo $(DISTRO) | grep -i ubuntu      >/dev/null 2>&1; echo $$?)
+    FEDORA = $(shell echo $(DISTRO) | grep -i fedora      >/dev/null 2>&1; echo $$?)
+    RHEL   = $(shell echo $(DISTRO) | grep -i 'red\|rhel' >/dev/null 2>&1; echo $$?)
+    CENTOS = $(shell echo $(DISTRO) | grep -i centos      >/dev/null 2>&1; echo $$?)
+    SUSE   = $(shell echo $(DISTRO) | grep -i 'suse\|sles' >/dev/null 2>&1; echo $$?)
+    ifeq ("$(UBUNTU)","0")
+      ifeq ($(HOST_ARCH)-$(TARGET_ARCH),x86_64-armv7l)
+        GLPATH := /usr/arm-linux-gnueabihf/lib
+        GLLINK := -L/usr/arm-linux-gnueabihf/lib
+        ifneq ($(TARGET_FS),) 
+          GLPATH += $(TARGET_FS)/usr/lib/arm-linux-gnueabihf
+          GLLINK += -L$(TARGET_FS)/usr/lib/arm-linux-gnueabihf
+        endif
+      else ifeq ($(HOST_ARCH)-$(TARGET_ARCH),x86_64-aarch64)
+        GLPATH := /usr/aarch64-linux-gnu/lib
+        GLLINK := -L/usr/aarch64-linux-gnu/lib
+        ifneq ($(TARGET_FS),)
+          GLPATH += $(TARGET_FS)/usr/lib
+          GLPATH += $(TARGET_FS)/usr/lib/aarch64-linux-gnu
+          GLLINK += -L$(TARGET_FS)/usr/lib/aarch64-linux-gnu
+        endif 
+      else
+        UBUNTU_PKG_NAME = $(shell which dpkg >/dev/null 2>&1 && dpkg -l 'nvidia-*' | grep '^ii' | awk '{print $$2}' | head -1)
+        ifneq ("$(UBUNTU_PKG_NAME)","")
+          GLPATH    ?= /usr/lib/$(UBUNTU_PKG_NAME)
+          GLLINK    ?= -L/usr/lib/$(UBUNTU_PKG_NAME)
+        endif
+
+        DFLT_PATH ?= /usr/lib
+      endif
+    endif
+    ifeq ("$(SUSE)","0")
+      GLPATH    ?= /usr/X11R6/lib64
+      GLLINK    ?= -L/usr/X11R6/lib64
+      DFLT_PATH ?= /usr/lib64
+    endif
+    ifeq ("$(FEDORA)","0")
+      GLPATH    ?= /usr/lib64/nvidia
+      GLLINK    ?= -L/usr/lib64/nvidia
+      DFLT_PATH ?= /usr/lib64
+    endif
+    ifeq ("$(RHEL)","0")
+      GLPATH    ?= /usr/lib64/nvidia
+      GLLINK    ?= -L/usr/lib64/nvidia
+      DFLT_PATH ?= /usr/lib64
+    endif
+    ifeq ("$(CENTOS)","0")
+      GLPATH    ?= /usr/lib64/nvidia
+      GLLINK    ?= -L/usr/lib64/nvidia
+      DFLT_PATH ?= /usr/lib64
+    endif
+
+  EGLLIB  := $(shell find -L $(GLPATH) $(DFLT_PATH) -name libEGL.so    -print 2>/dev/null)
+
+  ifeq ("$(EGLLIB)","")
+      $(info >>> WARNING - libEGL.so not found, please install libEGL.so <<<)
+      SAMPLE_ENABLED := 0
+  endif
+
+  HEADER_SEARCH_PATH ?= $(TARGET_FS)/usr/include
+  ifeq ($(HOST_ARCH)-$(TARGET_ARCH)-$(TARGET_OS),x86_64-armv7l-linux)
+      HEADER_SEARCH_PATH += /usr/arm-linux-gnueabihf/include
+  else ifeq ($(HOST_ARCH)-$(TARGET_ARCH)-$(TARGET_OS),x86_64-aarch64-linux)
+      HEADER_SEARCH_PATH += /usr/aarch64-linux-gnu/include
+  endif
+
+  EGLHEADER  := $(shell find -L $(HEADER_SEARCH_PATH) -name egl.h -print 2>/dev/null)
+  EGLEXTHEADER  := $(shell find -L $(HEADER_SEARCH_PATH) -name eglext.h -print 2>/dev/null)
+
+  ifeq ("$(EGLHEADER)","")
+      $(info >>> WARNING - egl.h not found, please install egl.h <<<)
+      SAMPLE_ENABLED := 0
+  endif
+  ifeq ("$(EGLEXTHEADER)","")
+      $(info >>> WARNING - eglext.h not found, please install eglext.h <<<)
+      SAMPLE_ENABLED := 0
+  endif
+else
+endif
+
+ifeq ("$(TARGET_OS)","qnx")
+    HOST_CCFLAGS := -V5.4.0,gcc_ntoaarch64le
+endif
+
+# Attempt to compile a minimal EGL application and run to check if EGL_SUPPORT_REUSE_NV is supported in the EGL headers available.
+ifneq ($(SAMPLE_ENABLED), 0)
+      $(shell printf "#include <EGL/egl.h>\n#include <EGL/eglext.h>\nint main() {\n#ifdef EGL_SUPPORT_REUSE_NV \n #error \"Compatible EGL header found\" \n  return 0;\n#endif \n return 1;\n}"  > test.c; )
+      EGL_DEFINES := $(shell $(HOST_COMPILER) $(HOST_CCFLAGS) $(CCFLAGS) $(EXTRA_CCFLAGS) -lEGL test.c -c 2>&1 | grep -ic "Compatible EGL header found";)
+      SHOULD_WAIVE := 0
+      ifeq ($(EGL_DEFINES),0)
+        SHOULD_WAIVE := 1
+      endif
+      ifeq ($(SHOULD_WAIVE),1)
+          $(info -----------------------------------------------------------------------------------------------)
+          $(info WARNING - NVIDIA EGL EXTENSIONS are not available in the present EGL headers)
+          $(info -----------------------------------------------------------------------------------------------)
+          $(info   This CUDA Sample cannot be built if the EGL NVIDIA EXTENSIONS like EGL_SUPPORT_REUSE_NV are not supported in EGL headers.)
+          $(info   This will be a dry-run of the Makefile.)
+          $(info   Please install the latest khronos EGL headers and libs to build this sample)
+          $(info -----------------------------------------------------------------------------------------------)
+          SAMPLE_ENABLED := 0
+      endif
+      $(shell rm test.o test.c 2>/dev/null)
+endif
+

Samples/EGLStream_CUDA_CrossGPU/helper.h

@@ -0,0 +1,221 @@
+/* Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "eglstrm_common.h"
+#if defined(EXTENSION_LIST)
+EXTENSION_LIST(EXTLST_EXTERN)
+#endif
+#include <cuda.h>
+
+int parseCmdLine(int argc, char *argv[], TestArgs *args);
+void printUsage(void);
+int NUMTRIALS = 10;
+int profileAPIs = 0;
+
+bool verbose = 0;
+bool isCrossDevice = 0;
+
+// Parse the command line options. Returns FAILURE on a parse error, SUCCESS
+// otherwise.
+int parseCmdLine(int argc, char *argv[], TestArgs *args) {
+  int i;
+
+  for (i = 1; i < argc; i++) {
+    if (strcmp(argv[i], "-h") == 0) {
+      printUsage();
+      exit(0);
+    } else if (strcmp(argv[i], "-n") == 0) {
+      ++i;
+      if (sscanf(argv[i], "%d", &NUMTRIALS) != 1 || NUMTRIALS <= 0) {
+        printf("Invalid trial count: %s should be > 0\n", argv[i]);
+        return -1;
+      }
+    } else if (strcmp(argv[i], "-profile") == 0) {
+      profileAPIs = 1;
+    } else if (strcmp(argv[i], "-crossdev") == 0) {
+      isCrossDevice = 1;
+    } else if (strcmp(argv[i], "-width") == 0) {
+      ++i;
+      if (sscanf(argv[i], "%d", &WIDTH) != 1 || (WIDTH <= 0)) {
+        printf("Width should be greater than 0\n");
+        return -1;
+      }
+    } else if (strcmp(argv[i], "-height") == 0) {
+      ++i;
+      if (sscanf(argv[i], "%d", &HEIGHT) != 1 || (HEIGHT <= 0)) {
+        printf("Width should be greater than 0\n");
+        return -1;
+      }
+    } else if (0 == strcmp(&argv[i][1], "proctype")) {
+      ++i;
+      if (!strcasecmp(argv[i], "prod")) {
+        args->isProducer = 1;
+      } else if (!strcasecmp(argv[i], "cons")) {
+        args->isProducer = 0;
+      } else {
+        printf("%s: Bad Process Type: %s\n", __func__, argv[i]);
+        return 1;
+      }
+    } else if (strcmp(argv[i], "-v") == 0) {
+      verbose = 1;
+    } else {
+      printf("Unknown option: %s\n", argv[i]);
+      return -1;
+    }
+  }
+
+  if (isCrossDevice) {
+    int deviceCount = 0;
+
+    CUresult error_id = cuInit(0);
+    if (error_id != CUDA_SUCCESS) {
+      printf("cuInit(0) returned %d\n", error_id);
+      printf("Result = FAIL\n");
+      exit(EXIT_FAILURE);
+    }
+
+    error_id = cuDeviceGetCount(&deviceCount);
+    if (error_id != CUDA_SUCCESS) {
+      printf("cuDeviceGetCount returned %d\n", (int)error_id);
+      printf("Result = FAIL\n");
+      exit(EXIT_FAILURE);
+    }
+
+    int iGPUexists = 0;
+    CUdevice dev;
+    for (dev = 0; dev < deviceCount; ++dev) {
+      int integrated = 0;
+      CUresult error_result = cuDeviceGetAttribute(
+          &integrated, CU_DEVICE_ATTRIBUTE_INTEGRATED, dev);
+
+      if (error_result != CUDA_SUCCESS) {
+        printf("cuDeviceGetAttribute returned error : %d\n", (int)error_result);
+        exit(EXIT_FAILURE);
+      }
+
+      if (integrated) {
+        iGPUexists = 1;
+      }
+    }
+
+    if (!iGPUexists) {
+      printf("No Integrated GPU found in the system.\n");
+      printf(
+          "-crossdev option is only supported on systems with an Integrated "
+          "GPU and a Discrete GPU\n");
+      printf("Waiving the execution\n");
+      exit(EXIT_SUCCESS);
+    }
+  }
+
+  if (!eglSetupExtensions(isCrossDevice)) {
+    printf("SetupExtentions failed \n");
+    exit(EXIT_FAILURE);
+  }
+#define MAX_EGL_DEVICES 4
+  EGLDeviceEXT devices[MAX_EGL_DEVICES];
+  EGLint numDevices = 0;
+  EGLBoolean eglStatus =
+      eglQueryDevicesEXT(MAX_EGL_DEVICES, devices, &numDevices);
+  if (eglStatus != EGL_TRUE) {
+    printf("Error querying EGL devices\n");
+    exit(EXIT_FAILURE);
+  }
+
+  if (numDevices == 0) {
+    printf("No EGL devices found\n");
+    eglStatus = EGL_FALSE;
+    exit(2);  // EXIT_WAIVED
+  }
+
+  int egl_device_id = 0;
+  for (egl_device_id = 0; egl_device_id < numDevices; egl_device_id++) {
+    EGLAttrib cuda_device;
+    eglStatus = eglQueryDeviceAttribEXT(devices[egl_device_id],
+                                        EGL_CUDA_DEVICE_NV, &cuda_device);
+    if (eglStatus == EGL_TRUE) {
+      break;
+    }
+  }
+
+  if (egl_device_id >= numDevices) {
+    printf("No CUDA Capable EGL Device found.. Waiving execution\n");
+    exit(2);  // EXIT_WAIVED
+  }
+
+  if (isCrossDevice) {
+    if (numDevices == 1) {
+      printf(
+          "Found only one EGL device, cannot setup cross GPU streams. "
+          "Waiving\n");
+      eglStatus = EGL_FALSE;
+      exit(2);  // EXIT_WAIVED
+    }
+  }
+
+  return 0;
+}
+
+void launchProducer(TestArgs *args) {
+  /* Cross-process creation of producer */
+  char argsProducer[1024];
+  char str[256];
+
+  strcpy(argsProducer, "./EGLStream_CUDA_CrossGPU -proctype prod ");
+
+  if (isCrossDevice) {
+    sprintf(str, "-crossdev ");
+    strcat(argsProducer, str);
+  }
+
+  if (verbose) {
+    sprintf(str, "-v ");
+    strcat(argsProducer, str);
+  }
+
+  /*Make the process run in bg*/
+  strcat(argsProducer, "& ");
+
+  printf("\n%s: Crossproc Producer command: %s \n", __func__, argsProducer);
+
+  /*Create crossproc Producer*/
+  system(argsProducer);
+
+  /*Enable crossproc Consumer in the same process */
+  args->isProducer = 0;
+}
+
+void printUsage(void) {
+  printf("Usage:\n");
+  printf("  -h           Print this help message\n");
+  printf("  -n n         Exit after running n trials. Set to 10 by default\n");
+  printf(
+      "  -profile     Profile time taken by ReleaseAPI. Not set by default\n");
+  printf("  -crossdev    Run with producer on idgpu and consumer on dgpu\n");
+  printf("  -dgpu        (same as -crossdev, deprecated)\n");
+  printf("  -v           verbose output\n");
+}

Samples/EGLStream_CUDA_CrossGPU/kernel.cu

@@ -0,0 +1,140 @@
+/* Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+//
+// DESCRIPTION:   Simple CUDA consumer rendering sample app
+//
+
+#include <EGL/egl.h>
+#include <EGL/eglext.h>
+#include <cuda.h>
+#include <cuda_runtime.h>
+#include <stdio.h>
+#include <string.h>
+
+#include "eglstrm_common.h"
+
+extern bool isCrossDevice;
+
+__device__ static unsigned int numErrors = 0, errorFound = 0;
+__device__ void checkProducerDataGPU(char *data, int size, char expectedVal,
+                                     int frameNumber) {
+  if ((data[blockDim.x * blockIdx.x + threadIdx.x] != expectedVal) &&
+      (!errorFound)) {
+    printf("Producer FOUND:%d expected: %d at %d for trial %d %d\n",
+           data[blockDim.x * blockIdx.x + threadIdx.x], expectedVal,
+           (blockDim.x * blockIdx.x + threadIdx.x), frameNumber, numErrors);
+    numErrors++;
+    errorFound = 1;
+    return;
+  }
+}
+
+__device__ void checkConsumerDataGPU(char *data, int size, char expectedVal,
+                                     int frameNumber) {
+  if ((data[blockDim.x * blockIdx.x + threadIdx.x] != expectedVal) &&
+      (!errorFound)) {
+    printf("Consumer FOUND:%d expected: %d at %d for trial %d %d\n",
+           data[blockDim.x * blockIdx.x + threadIdx.x], expectedVal,
+           (blockDim.x * blockIdx.x + threadIdx.x), frameNumber, numErrors);
+    numErrors++;
+    errorFound = 1;
+    return;
+  }
+}
+
+__global__ void writeDataToBuffer(char *pSrc, char newVal) {
+  pSrc[blockDim.x * blockIdx.x + threadIdx.x] = newVal;
+}
+
+__global__ void testKernelConsumer(char *pSrc, char size, char expectedVal,
+                                   char newVal, int frameNumber) {
+  checkConsumerDataGPU(pSrc, size, expectedVal, frameNumber);
+}
+
+__global__ void testKernelProducer(char *pSrc, char size, char expectedVal,
+                                   char newVal, int frameNumber) {
+  checkProducerDataGPU(pSrc, size, expectedVal, frameNumber);
+}
+__global__ void getNumErrors(int *numErr) { *numErr = numErrors; }
+
+cudaError_t cudaProducer_filter(cudaStream_t pStream, char *pSrc, int width,
+                                int height, char expectedVal, char newVal,
+                                int frameNumber) {
+  // in case where consumer is on dgpu and producer is on igpu when return is
+  // called the frame is not copied back to igpu. So the consumer changes is not
+  // visible to producer
+  if (isCrossDevice == 0) {
+    testKernelProducer<<<(width * height) / 1024, 1024, 1, pStream>>>(
+        pSrc, width * height, expectedVal, newVal, frameNumber);
+  }
+  writeDataToBuffer<<<(width * height) / 1024, 1024, 1, pStream>>>(pSrc,
+                                                                   newVal);
+  return cudaSuccess;
+};
+
+cudaError_t cudaConsumer_filter(cudaStream_t cStream, char *pSrc, int width,
+                                int height, char expectedVal, char newVal,
+                                int frameNumber) {
+  testKernelConsumer<<<(width * height) / 1024, 1024, 1, cStream>>>(
+      pSrc, width * height, expectedVal, newVal, frameNumber);
+  writeDataToBuffer<<<(width * height) / 1024, 1024, 1, cStream>>>(pSrc,
+                                                                   newVal);
+  return cudaSuccess;
+};
+
+cudaError_t cudaGetValueMismatch() {
+  int numErr_h;
+  int *numErr_d = NULL;
+  cudaError_t err = cudaSuccess;
+  err = cudaMalloc(&numErr_d, sizeof(int));
+  if (err != cudaSuccess) {
+    printf("Cuda Main: cudaMalloc failed with %s\n", cudaGetErrorString(err));
+    return err;
+  }
+  getNumErrors<<<1, 1>>>(numErr_d);
+  err = cudaDeviceSynchronize();
+  if (err != cudaSuccess) {
+    printf("Cuda Main: cudaDeviceSynchronize failed with %s\n",
+           cudaGetErrorString(err));
+  }
+  err = cudaMemcpy(&numErr_h, numErr_d, sizeof(int), cudaMemcpyDeviceToHost);
+  if (err != cudaSuccess) {
+    printf("Cuda Main: cudaMemcpy failed with %s\n", cudaGetErrorString(err));
+    cudaFree(numErr_d);
+    return err;
+  }
+  err = cudaFree(numErr_d);
+  if (err != cudaSuccess) {
+    printf("Cuda Main: cudaFree failed with %s\n", cudaGetErrorString(err));
+    return err;
+  }
+  if (numErr_h > 0) {
+    return cudaErrorUnknown;
+  }
+  return cudaSuccess;
+}

Samples/EGLStream_CUDA_CrossGPU/main.cpp

@@ -0,0 +1,392 @@
+/* Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "cudaEGL.h"
+#include "cuda_consumer.h"
+#include "cuda_producer.h"
+#include "eglstrm_common.h"
+#include "helper.h"
+#if defined(EXTENSION_LIST)
+EXTENSION_LIST(EXTLST_EXTERN)
+#endif
+
+bool signal_stop = 0;
+extern bool verbose;
+
+static void sig_handler(int sig) {
+  signal_stop = 1;
+  printf("Signal: %d\n", sig);
+}
+
+void DoneCons(int consumerStatus, int send_fd) {
+  EGLStreamFini();
+  // get the final status from producer, combine and print
+  int producerStatus = -1;
+  if (-1 == recv(send_fd, (void *)&producerStatus, sizeof(int), 0)) {
+    printf("%s: Cuda Consumer could not receive status from producer.\n",
+           __func__);
+  }
+  close(send_fd);
+
+  if (producerStatus == 0 && consumerStatus == 0) {
+    printf("&&&& EGLStream_CUDA_CrossGPU PASSED\n");
+    exit(EXIT_SUCCESS);
+  } else {
+    printf("&&&& EGLStream_CUDA_CrossGPU FAILED\n");
+    exit(EXIT_FAILURE);
+  }
+}
+
+void DoneProd(int producerStatus, int connect_fd) {
+  EGLStreamFini();
+  if (-1 == send(connect_fd, (void *)&producerStatus, sizeof(int), 0)) {
+    printf("%s: Cuda Producer could not send status to consumer.\n", __func__);
+  }
+  close(connect_fd);
+  if (producerStatus == 0) {
+    exit(EXIT_SUCCESS);
+  } else {
+    exit(EXIT_FAILURE);
+  }
+}
+
+int WIDTH = 8192, HEIGHT = 8192;
+int main(int argc, char **argv) {
+  TestArgs args = {0, false};
+  CUresult curesult = CUDA_SUCCESS;
+  unsigned int j = 0;
+  cudaError_t err = cudaSuccess;
+  EGLNativeFileDescriptorKHR fileDescriptor = EGL_NO_FILE_DESCRIPTOR_KHR;
+  struct timespec start, end;
+  CUeglFrame cudaEgl1, cudaEgl2;
+  int consumerStatus = 0;
+  int send_fd = -1;
+
+  if (parseCmdLine(argc, argv, &args) < 0) {
+    printUsage();
+    curesult = CUDA_ERROR_UNKNOWN;
+    DoneCons(consumerStatus, send_fd);
+  }
+
+  printf("Width : %u, height: %u and iterations: %u\n", WIDTH, HEIGHT,
+         NUMTRIALS);
+
+  if (!args.isProducer)  // Consumer code
+  {
+    test_cuda_consumer_s cudaConsumer;
+    memset(&cudaConsumer, 0, sizeof(test_cuda_consumer_s));
+    cudaConsumer.profileAPI = profileAPIs;
+
+    // Hook up Ctrl-C handler
+    signal(SIGINT, sig_handler);
+
+    if (!EGLStreamInit(isCrossDevice, !args.isProducer,
+                       EGL_NO_FILE_DESCRIPTOR_KHR)) {
+      printf("EGLStream Init failed.\n");
+      curesult = CUDA_ERROR_UNKNOWN;
+      consumerStatus = -1;
+      DoneCons(consumerStatus, send_fd);
+    }
+
+    cudaConsumer.cudaDevId = cudaDevIndexCons;
+    curesult = cudaDeviceCreateConsumer(&cudaConsumer);
+    if (curesult != CUDA_SUCCESS) {
+      consumerStatus = -1;
+      DoneCons(consumerStatus, send_fd);
+    }
+
+    cuCtxPushCurrent(cudaConsumer.context);
+
+    launchProducer(&args);
+
+    args.charCnt = WIDTH * HEIGHT * 4;
+
+    curesult = cuda_consumer_init(&cudaConsumer, &args);
+    if (curesult != CUDA_SUCCESS) {
+      printf("Cuda Consumer: Init failed, status: %d\n", curesult);
+      consumerStatus = -1;
+      DoneCons(consumerStatus, send_fd);
+    }
+
+    cuCtxPopCurrent(&cudaConsumer.context);
+
+    send_fd = UnixSocketConnect(SOCK_PATH);
+    if (-1 == send_fd) {
+      printf("%s: Cuda Consumer cannot create socket %s\n", __func__,
+             SOCK_PATH);
+      consumerStatus = -1;
+      DoneCons(consumerStatus, send_fd);
+    }
+
+    cuCtxPushCurrent(cudaConsumer.context);
+    cudaConsumer.eglStream = g_consumerEglStream;
+    cudaConsumer.eglDisplay = g_consumerEglDisplay;
+
+    // Send the EGL stream FD to producer
+    fileDescriptor = eglGetStreamFileDescriptorKHR(cudaConsumer.eglDisplay,
+                                                   cudaConsumer.eglStream);
+    if (EGL_NO_FILE_DESCRIPTOR_KHR == fileDescriptor) {
+      printf("%s: Cuda Consumer could not get EGL file descriptor.\n",
+             __func__);
+      eglDestroyStreamKHR(cudaConsumer.eglDisplay, cudaConsumer.eglStream);
+      consumerStatus = -1;
+      DoneCons(consumerStatus, send_fd);
+    }
+
+    if (verbose)
+      printf("%s: Cuda Consumer EGL stream FD obtained : %d.\n", __func__,
+             fileDescriptor);
+
+    int res = -1;
+    res = EGLStreamSendfd(send_fd, fileDescriptor);
+    if (-1 == res) {
+      printf("%s: Cuda Consumer could not send EGL file descriptor.\n",
+             __func__);
+      consumerStatus = -1;
+      close(fileDescriptor);
+    }
+
+    if (CUDA_SUCCESS !=
+        (curesult = cuEGLStreamConsumerConnect(&(cudaConsumer.cudaConn),
+                                               cudaConsumer.eglStream))) {
+      printf("FAILED Connect CUDA consumer with error %d\n", curesult);
+      consumerStatus = -1;
+      DoneCons(consumerStatus, send_fd);
+    }
+
+    j = 0;
+    for (j = 0; j < NUMTRIALS; j++) {
+      curesult = cudaConsumerAcquireFrame(&cudaConsumer, j);
+      if (curesult != CUDA_SUCCESS) {
+        printf("Cuda Consumer Test failed for frame = %d\n", j + 1);
+        consumerStatus = -1;
+        DoneCons(consumerStatus, send_fd);
+      }
+      curesult = cudaConsumerReleaseFrame(&cudaConsumer, j);
+      if (curesult != CUDA_SUCCESS) {
+        printf("Cuda Consumer Test failed for frame = %d\n", j + 1);
+        consumerStatus = -1;
+        DoneCons(consumerStatus, send_fd);
+      }
+
+      curesult = cudaConsumerAcquireFrame(&cudaConsumer, j);
+      if (curesult != CUDA_SUCCESS) {
+        printf("Cuda Consumer Test failed for frame = %d\n", j + 1);
+        consumerStatus = -1;
+        DoneCons(consumerStatus, send_fd);
+      }
+      curesult = cudaConsumerReleaseFrame(&cudaConsumer, j);
+      if (curesult != CUDA_SUCCESS) {
+        printf("Cuda Consumer Test failed for frame = %d\n", j + 1);
+        consumerStatus = -1;
+        DoneCons(consumerStatus, send_fd);
+      }
+    }
+    cuCtxSynchronize();
+    close(fileDescriptor);
+    err = cudaGetValueMismatch();
+    if (err != cudaSuccess) {
+      printf("Consumer: App failed with value mismatch\n");
+      curesult = CUDA_ERROR_UNKNOWN;
+      consumerStatus = -1;
+      DoneCons(consumerStatus, send_fd);
+    }
+
+    EGLint streamState = 0;
+    if (!eglQueryStreamKHR(cudaConsumer.eglDisplay, cudaConsumer.eglStream,
+                           EGL_STREAM_STATE_KHR, &streamState)) {
+      printf("Main, eglQueryStreamKHR EGL_STREAM_STATE_KHR failed\n");
+      curesult = CUDA_ERROR_UNKNOWN;
+      consumerStatus = -1;
+      DoneCons(consumerStatus, send_fd);
+    }
+
+    if (streamState != EGL_STREAM_STATE_DISCONNECTED_KHR) {
+      if (CUDA_SUCCESS != (curesult = cuda_consumer_Deinit(&cudaConsumer))) {
+        printf("Consumer Disconnect FAILED.\n");
+        consumerStatus = -1;
+        DoneCons(consumerStatus, send_fd);
+      }
+    }
+  } else  // Producer
+  {
+    test_cuda_producer_s cudaProducer;
+    memset(&cudaProducer, 0, sizeof(test_cuda_producer_s));
+    cudaProducer.profileAPI = profileAPIs;
+    int producerStatus = 0;
+
+    setenv("CUDA_EGL_PRODUCER_RETURN_WAIT_TIMEOUT", "1600", 0);
+
+    int connect_fd = -1;
+    // Hook up Ctrl-C handler
+    signal(SIGINT, sig_handler);
+
+    // Create connection to Consumer
+    connect_fd = UnixSocketCreate(SOCK_PATH);
+    if (-1 == connect_fd) {
+      printf("%s: Cuda Producer could not create socket: %s.\n", __func__,
+             SOCK_PATH);
+      producerStatus = -1;
+      DoneProd(producerStatus, connect_fd);
+    }
+
+    // Get the file descriptor of the stream from the consumer process
+    // and re-create the EGL stream from it
+    fileDescriptor = EGLStreamReceivefd(connect_fd);
+    if (-1 == fileDescriptor) {
+      printf("%s: Cuda Producer could not receive EGL file descriptor \n",
+             __func__);
+      producerStatus = -1;
+      DoneProd(producerStatus, connect_fd);
+    }
+
+    if (!EGLStreamInit(isCrossDevice, 0, fileDescriptor)) {
+      printf("EGLStream Init failed.\n");
+      producerStatus = -1;
+      curesult = CUDA_ERROR_UNKNOWN;
+      DoneProd(producerStatus, connect_fd);
+    }
+
+    cudaProducer.eglDisplay = g_producerEglDisplay;
+    cudaProducer.eglStream = g_producerEglStream;
+    cudaProducer.cudaDevId = cudaDevIndexProd;
+
+    curesult = cudaDeviceCreateProducer(&cudaProducer);
+    if (curesult != CUDA_SUCCESS) {
+      producerStatus = -1;
+      DoneProd(producerStatus, connect_fd);
+    }
+
+    args.charCnt = WIDTH * HEIGHT * 4;
+    cuCtxPushCurrent(cudaProducer.context);
+    curesult = cudaProducerInit(&cudaProducer, &args);
+    if (curesult != CUDA_SUCCESS) {
+      printf("Cuda Producer: Init failed, status: %d\n", curesult);
+      producerStatus = -1;
+      DoneProd(producerStatus, connect_fd);
+    }
+
+    // wait for consumer to connect first
+    int err = 0;
+    int wait_loop = 0;
+    EGLint streamState = 0;
+    do {
+      err = eglQueryStreamKHR(cudaProducer.eglDisplay, cudaProducer.eglStream,
+                              EGL_STREAM_STATE_KHR, &streamState);
+      if ((0 != err) && (EGL_STREAM_STATE_CONNECTING_KHR != streamState)) {
+        sleep(1);
+        wait_loop++;
+      }
+    } while ((wait_loop < 10) && (0 != err) &&
+             (streamState != EGL_STREAM_STATE_CONNECTING_KHR));
+
+    if ((0 == err) || (wait_loop >= 10)) {
+      printf(
+          "%s: Cuda Producer eglQueryStreamKHR EGL_STREAM_STATE_KHR failed.\n",
+          __func__);
+      producerStatus = -1;
+      DoneProd(producerStatus, connect_fd);
+    }
+
+    if (CUDA_SUCCESS != (curesult = cuEGLStreamProducerConnect(
+                             &(cudaProducer.cudaConn), cudaProducer.eglStream,
+                             WIDTH, HEIGHT))) {
+      printf("Connect CUDA producer FAILED with error %d\n", curesult);
+      producerStatus = -1;
+      DoneProd(producerStatus, connect_fd);
+    }
+
+    printf("main - Cuda Producer and Consumer Initialized.\n");
+
+    cudaProducerPrepareFrame(&cudaEgl1, cudaProducer.cudaPtr, args.charCnt);
+    cudaProducerPrepareFrame(&cudaEgl2, cudaProducer.cudaPtr1, args.charCnt);
+
+    j = 0;
+    for (j = 0; j < NUMTRIALS; j++) {
+      curesult = cudaProducerPresentFrame(&cudaProducer, cudaEgl1, j);
+      if (curesult != CUDA_SUCCESS) {
+        printf("Cuda Producer Test failed for frame = %d with cuda error:%d\n",
+               j + 1, curesult);
+        producerStatus = -1;
+        DoneProd(producerStatus, connect_fd);
+      }
+
+      curesult = cudaProducerPresentFrame(&cudaProducer, cudaEgl2, j);
+      if (curesult != CUDA_SUCCESS) {
+        printf("Cuda Producer Test failed for frame = %d with cuda error:%d\n",
+               j + 1, curesult);
+        producerStatus = -1;
+        DoneProd(producerStatus, connect_fd);
+      }
+
+      curesult = cudaProducerReturnFrame(&cudaProducer, cudaEgl1, j);
+      if (curesult != CUDA_SUCCESS) {
+        printf("Cuda Producer Test failed for frame = %d with cuda error:%d\n",
+               j + 1, curesult);
+        producerStatus = -1;
+        DoneProd(producerStatus, connect_fd);
+      }
+
+      curesult = cudaProducerReturnFrame(&cudaProducer, cudaEgl2, j);
+      if (curesult != CUDA_SUCCESS) {
+        printf("Cuda Producer Test failed for frame = %d with cuda error:%d\n",
+               j + 1, curesult);
+        producerStatus = -1;
+        DoneProd(producerStatus, connect_fd);
+      }
+    }
+
+    cuCtxSynchronize();
+    err = cudaGetValueMismatch();
+    if (err != cudaSuccess) {
+      printf("Prod: App failed with value mismatch\n");
+      curesult = CUDA_ERROR_UNKNOWN;
+      producerStatus = -1;
+      DoneProd(producerStatus, connect_fd);
+    }
+
+    printf("Tear Down Start.....\n");
+    if (!eglQueryStreamKHR(cudaProducer.eglDisplay, cudaProducer.eglStream,
+                           EGL_STREAM_STATE_KHR, &streamState)) {
+      printf("Main, eglQueryStreamKHR EGL_STREAM_STATE_KHR failed\n");
+      curesult = CUDA_ERROR_UNKNOWN;
+      producerStatus = -1;
+      DoneProd(producerStatus, connect_fd);
+    }
+
+    if (streamState != EGL_STREAM_STATE_DISCONNECTED_KHR) {
+      if (CUDA_SUCCESS != (curesult = cudaProducerDeinit(&cudaProducer))) {
+        printf("Producer Disconnect FAILED with %d\n", curesult);
+        producerStatus = -1;
+        DoneProd(producerStatus, connect_fd);
+      }
+    }
+    unsetenv("CUDA_EGL_PRODUCER_RETURN_WAIT_TIMEOUT");
+  }
+
+  return 0;
+}