add and update samples for CUDA 11.6

Samples/4_CUDA_Libraries/cuSolverRf/.vscode/c_cpp_properties.json

@@ -0,0 +1,18 @@
+{
+    "configurations": [
+        {
+            "name": "Linux",
+            "includePath": [
+                "${workspaceFolder}/**",
+                "${workspaceFolder}/../../../Common"
+            ],
+            "defines": [],
+            "compilerPath": "/usr/local/cuda/bin/nvcc",
+            "cStandard": "gnu17",
+            "cppStandard": "gnu++14",
+            "intelliSenseMode": "linux-gcc-x64",
+            "configurationProvider": "ms-vscode.makefile-tools"
+        }
+    ],
+    "version": 4
+}

Samples/4_CUDA_Libraries/cuSolverRf/.vscode/extensions.json

@@ -0,0 +1,7 @@
+{
+    "recommendations": [
+        "nvidia.nsight-vscode-edition",
+        "ms-vscode.cpptools",
+        "ms-vscode.makefile-tools"
+    ]
+}

Samples/4_CUDA_Libraries/cuSolverRf/.vscode/launch.json

@@ -0,0 +1,10 @@
+{
+    "configurations": [
+        {
+            "name": "CUDA C++: Launch",
+            "type": "cuda-gdb",
+            "request": "launch",
+            "program": "${workspaceFolder}/cuSolverRf"
+        }
+    ]
+}

Samples/4_CUDA_Libraries/cuSolverRf/.vscode/tasks.json

@@ -0,0 +1,15 @@
+{
+    "version": "2.0.0",
+    "tasks": [
+        {
+            "label": "sample",
+            "type": "shell",
+            "command": "make dbg=1",
+            "problemMatcher": ["$nvcc"],
+            "group": {
+                "kind": "build",
+                "isDefault": true
+            }
+        }
+    ]
+}

Samples/4_CUDA_Libraries/cuSolverRf/Makefile

@@ -0,0 +1,354 @@
+################################################################################
+# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#  * Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#  * Neither the name of NVIDIA CORPORATION nor the names of its
+#    contributors may be used to endorse or promote products derived
+#    from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+################################################################################
+#
+# Makefile project only supported on Mac OS X and Linux Platforms)
+#
+################################################################################
+
+# Location of the CUDA Toolkit
+CUDA_PATH ?= /usr/local/cuda
+
+##############################
+# start deprecated interface #
+##############################
+ifeq ($(x86_64),1)
+    $(info WARNING - x86_64 variable has been deprecated)
+    $(info WARNING - please use TARGET_ARCH=x86_64 instead)
+    TARGET_ARCH ?= x86_64
+endif
+ifeq ($(ARMv7),1)
+    $(info WARNING - ARMv7 variable has been deprecated)
+    $(info WARNING - please use TARGET_ARCH=armv7l instead)
+    TARGET_ARCH ?= armv7l
+endif
+ifeq ($(aarch64),1)
+    $(info WARNING - aarch64 variable has been deprecated)
+    $(info WARNING - please use TARGET_ARCH=aarch64 instead)
+    TARGET_ARCH ?= aarch64
+endif
+ifeq ($(ppc64le),1)
+    $(info WARNING - ppc64le variable has been deprecated)
+    $(info WARNING - please use TARGET_ARCH=ppc64le instead)
+    TARGET_ARCH ?= ppc64le
+endif
+ifneq ($(GCC),)
+    $(info WARNING - GCC variable has been deprecated)
+    $(info WARNING - please use HOST_COMPILER=$(GCC) instead)
+    HOST_COMPILER ?= $(GCC)
+endif
+ifneq ($(abi),)
+    $(error ERROR - abi variable has been removed)
+endif
+############################
+# end deprecated interface #
+############################
+
+# architecture
+HOST_ARCH   := $(shell uname -m)
+TARGET_ARCH ?= $(HOST_ARCH)
+ifneq (,$(filter $(TARGET_ARCH),x86_64 aarch64 sbsa ppc64le armv7l))
+    ifneq ($(TARGET_ARCH),$(HOST_ARCH))
+        ifneq (,$(filter $(TARGET_ARCH),x86_64 aarch64 sbsa ppc64le))
+            TARGET_SIZE := 64
+        else ifneq (,$(filter $(TARGET_ARCH),armv7l))
+            TARGET_SIZE := 32
+        endif
+    else
+        TARGET_SIZE := $(shell getconf LONG_BIT)
+    endif
+else
+    $(error ERROR - unsupported value $(TARGET_ARCH) for TARGET_ARCH!)
+endif
+
+# sbsa and aarch64 systems look similar. Need to differentiate them at host level for now.
+ifeq ($(HOST_ARCH),aarch64)
+    ifeq ($(CUDA_PATH)/targets/sbsa-linux,$(shell ls -1d $(CUDA_PATH)/targets/sbsa-linux 2>/dev/null))
+        HOST_ARCH := sbsa
+        TARGET_ARCH := sbsa
+    endif
+endif
+
+ifneq ($(TARGET_ARCH),$(HOST_ARCH))
+    ifeq (,$(filter $(HOST_ARCH)-$(TARGET_ARCH),aarch64-armv7l x86_64-armv7l x86_64-aarch64 x86_64-sbsa x86_64-ppc64le))
+        $(error ERROR - cross compiling from $(HOST_ARCH) to $(TARGET_ARCH) is not supported!)
+    endif
+endif
+
+# When on native aarch64 system with userspace of 32-bit, change TARGET_ARCH to armv7l
+ifeq ($(HOST_ARCH)-$(TARGET_ARCH)-$(TARGET_SIZE),aarch64-aarch64-32)
+    TARGET_ARCH = armv7l
+endif
+
+# operating system
+HOST_OS   := $(shell uname -s 2>/dev/null | tr "[:upper:]" "[:lower:]")
+TARGET_OS ?= $(HOST_OS)
+ifeq (,$(filter $(TARGET_OS),linux darwin qnx android))
+    $(error ERROR - unsupported value $(TARGET_OS) for TARGET_OS!)
+endif
+
+# host compiler
+ifeq ($(TARGET_OS),darwin)
+    ifeq ($(shell expr `xcodebuild -version | grep -i xcode | awk '{print $$2}' | cut -d'.' -f1` \>= 5),1)
+        HOST_COMPILER ?= clang++
+    endif
+else ifneq ($(TARGET_ARCH),$(HOST_ARCH))
+    ifeq ($(HOST_ARCH)-$(TARGET_ARCH),x86_64-armv7l)
+        ifeq ($(TARGET_OS),linux)
+            HOST_COMPILER ?= arm-linux-gnueabihf-g++
+        else ifeq ($(TARGET_OS),qnx)
+            ifeq ($(QNX_HOST),)
+                $(error ERROR - QNX_HOST must be passed to the QNX host toolchain)
+            endif
+            ifeq ($(QNX_TARGET),)
+                $(error ERROR - QNX_TARGET must be passed to the QNX target toolchain)
+            endif
+            export QNX_HOST
+            export QNX_TARGET
+            HOST_COMPILER ?= $(QNX_HOST)/usr/bin/arm-unknown-nto-qnx6.6.0eabi-g++
+        else ifeq ($(TARGET_OS),android)
+            HOST_COMPILER ?= arm-linux-androideabi-g++
+        endif
+    else ifeq ($(TARGET_ARCH),aarch64)
+        ifeq ($(TARGET_OS), linux)
+            HOST_COMPILER ?= aarch64-linux-gnu-g++
+        else ifeq ($(TARGET_OS),qnx)
+            ifeq ($(QNX_HOST),)
+                $(error ERROR - QNX_HOST must be passed to the QNX host toolchain)
+            endif
+            ifeq ($(QNX_TARGET),)
+                $(error ERROR - QNX_TARGET must be passed to the QNX target toolchain)
+            endif
+            export QNX_HOST
+            export QNX_TARGET
+            HOST_COMPILER ?= $(QNX_HOST)/usr/bin/q++
+        else ifeq ($(TARGET_OS), android)
+            HOST_COMPILER ?= aarch64-linux-android-clang++
+        endif
+    else ifeq ($(TARGET_ARCH),sbsa)
+        HOST_COMPILER ?= aarch64-linux-gnu-g++
+    else ifeq ($(TARGET_ARCH),ppc64le)
+        HOST_COMPILER ?= powerpc64le-linux-gnu-g++
+    endif
+endif
+HOST_COMPILER ?= g++
+NVCC          := $(CUDA_PATH)/bin/nvcc -ccbin $(HOST_COMPILER)
+
+# internal flags
+NVCCFLAGS   := -m${TARGET_SIZE}
+CCFLAGS     :=
+LDFLAGS     :=
+
+# build flags
+ifeq ($(TARGET_OS),darwin)
+    LDFLAGS += -rpath $(CUDA_PATH)/lib
+    CCFLAGS += -arch $(HOST_ARCH)
+else ifeq ($(HOST_ARCH)-$(TARGET_ARCH)-$(TARGET_OS),x86_64-armv7l-linux)
+    LDFLAGS += --dynamic-linker=/lib/ld-linux-armhf.so.3
+    CCFLAGS += -mfloat-abi=hard
+else ifeq ($(TARGET_OS),android)
+    LDFLAGS += -pie
+    CCFLAGS += -fpie -fpic -fexceptions
+endif
+
+ifneq ($(TARGET_ARCH),$(HOST_ARCH))
+    ifeq ($(TARGET_ARCH)-$(TARGET_OS),armv7l-linux)
+        ifneq ($(TARGET_FS),)
+            GCCVERSIONLTEQ46 := $(shell expr `$(HOST_COMPILER) -dumpversion` \<= 4.6)
+            ifeq ($(GCCVERSIONLTEQ46),1)
+                CCFLAGS += --sysroot=$(TARGET_FS)
+            endif
+            LDFLAGS += --sysroot=$(TARGET_FS)
+            LDFLAGS += -rpath-link=$(TARGET_FS)/lib
+            LDFLAGS += -rpath-link=$(TARGET_FS)/usr/lib
+            LDFLAGS += -rpath-link=$(TARGET_FS)/usr/lib/arm-linux-gnueabihf
+        endif
+    endif
+    ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-linux)
+        ifneq ($(TARGET_FS),)
+            GCCVERSIONLTEQ46 := $(shell expr `$(HOST_COMPILER) -dumpversion` \<= 4.6)
+            ifeq ($(GCCVERSIONLTEQ46),1)
+                CCFLAGS += --sysroot=$(TARGET_FS)
+            endif
+            LDFLAGS += --sysroot=$(TARGET_FS)
+            LDFLAGS += -rpath-link=$(TARGET_FS)/lib -L$(TARGET_FS)/lib
+            LDFLAGS += -rpath-link=$(TARGET_FS)/lib/aarch64-linux-gnu -L$(TARGET_FS)/lib/aarch64-linux-gnu
+            LDFLAGS += -rpath-link=$(TARGET_FS)/usr/lib -L$(TARGET_FS)/usr/lib
+            LDFLAGS += -rpath-link=$(TARGET_FS)/usr/lib/aarch64-linux-gnu -L$(TARGET_FS)/usr/lib/aarch64-linux-gnu
+            LDFLAGS += --unresolved-symbols=ignore-in-shared-libs
+            CCFLAGS += -isystem=$(TARGET_FS)/usr/include -I$(TARGET_FS)/usr/include -I$(TARGET_FS)/usr/include/libdrm
+            CCFLAGS += -isystem=$(TARGET_FS)/usr/include/aarch64-linux-gnu -I$(TARGET_FS)/usr/include/aarch64-linux-gnu
+        endif
+    endif
+    ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-qnx)
+        NVCCFLAGS += -D_QNX_SOURCE
+        NVCCFLAGS += --qpp-config 8.3.0,gcc_ntoaarch64le
+        CCFLAGS += -DWIN_INTERFACE_CUSTOM -I/usr/include/aarch64-qnx-gnu
+        LDFLAGS += -lsocket
+        LDFLAGS += -L/usr/lib/aarch64-qnx-gnu
+        CCFLAGS += "-Wl\,-rpath-link\,/usr/lib/aarch64-qnx-gnu"
+        ifdef TARGET_OVERRIDE
+            LDFLAGS += -lslog2
+        endif
+
+        ifneq ($(TARGET_FS),)
+            LDFLAGS += -L$(TARGET_FS)/usr/lib
+            CCFLAGS += "-Wl\,-rpath-link\,$(TARGET_FS)/usr/lib"
+            LDFLAGS += -L$(TARGET_FS)/usr/libnvidia
+            CCFLAGS += "-Wl\,-rpath-link\,$(TARGET_FS)/usr/libnvidia"
+            CCFLAGS += -I$(TARGET_FS)/../include
+        endif
+    endif
+endif
+
+ifdef TARGET_OVERRIDE # cuda toolkit targets override
+    NVCCFLAGS += -target-dir $(TARGET_OVERRIDE)
+endif
+
+# Install directory of different arch
+CUDA_INSTALL_TARGET_DIR :=
+ifeq ($(TARGET_ARCH)-$(TARGET_OS),armv7l-linux)
+    CUDA_INSTALL_TARGET_DIR = targets/armv7-linux-gnueabihf/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-linux)
+    CUDA_INSTALL_TARGET_DIR = targets/aarch64-linux/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),sbsa-linux)
+    CUDA_INSTALL_TARGET_DIR = targets/sbsa-linux/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),armv7l-android)
+    CUDA_INSTALL_TARGET_DIR = targets/armv7-linux-androideabi/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-android)
+    CUDA_INSTALL_TARGET_DIR = targets/aarch64-linux-androideabi/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),armv7l-qnx)
+    CUDA_INSTALL_TARGET_DIR = targets/ARMv7-linux-QNX/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-qnx)
+    CUDA_INSTALL_TARGET_DIR = targets/aarch64-qnx/
+else ifeq ($(TARGET_ARCH),ppc64le)
+    CUDA_INSTALL_TARGET_DIR = targets/ppc64le-linux/
+endif
+
+# Debug build flags
+ifeq ($(dbg),1)
+      NVCCFLAGS += -g -G
+      BUILD_TYPE := debug
+else
+      BUILD_TYPE := release
+endif
+
+ALL_CCFLAGS :=
+ALL_CCFLAGS += $(NVCCFLAGS)
+ALL_CCFLAGS += $(EXTRA_NVCCFLAGS)
+ALL_CCFLAGS += $(addprefix -Xcompiler ,$(CCFLAGS))
+ALL_CCFLAGS += $(addprefix -Xcompiler ,$(EXTRA_CCFLAGS))
+
+SAMPLE_ENABLED := 1
+
+ifeq ($(TARGET_OS),linux)
+ALL_CCFLAGS += -Xcompiler \"-Wl,--no-as-needed\"
+endif
+
+ALL_LDFLAGS :=
+ALL_LDFLAGS += $(ALL_CCFLAGS)
+ALL_LDFLAGS += $(addprefix -Xlinker ,$(LDFLAGS))
+ALL_LDFLAGS += $(addprefix -Xlinker ,$(EXTRA_LDFLAGS))
+
+# Common includes and paths for CUDA
+INCLUDES  := -I../../../Common
+LIBRARIES :=
+
+################################################################################
+
+# Gencode arguments
+ifeq ($(TARGET_ARCH),$(filter $(TARGET_ARCH),armv7l aarch64 sbsa))
+SMS ?= 53 61 70 72 75 80 86 87
+else
+SMS ?= 35 37 50 52 60 61 70 75 80 86
+endif
+
+ifeq ($(SMS),)
+$(info >>> WARNING - no SM architectures have been specified - waiving sample <<<)
+SAMPLE_ENABLED := 0
+endif
+
+ifeq ($(GENCODE_FLAGS),)
+# Generate SASS code for each SM architecture listed in $(SMS)
+$(foreach sm,$(SMS),$(eval GENCODE_FLAGS += -gencode arch=compute_$(sm),code=sm_$(sm)))
+
+# Generate PTX code from the highest SM architecture in $(SMS) to guarantee forward-compatibility
+HIGHEST_SM := $(lastword $(sort $(SMS)))
+ifneq ($(HIGHEST_SM),)
+GENCODE_FLAGS += -gencode arch=compute_$(HIGHEST_SM),code=compute_$(HIGHEST_SM)
+endif
+endif
+
+ALL_CCFLAGS += --threads 0 --std=c++11
+
+LIBRARIES += -lcusolver -lcublas -lcusparse
+
+ifeq ($(SAMPLE_ENABLED),0)
+EXEC ?= @echo "[@]"
+endif
+
+################################################################################
+
+# Target rules
+all: build
+
+build: cuSolverRf
+
+check.deps:
+ifeq ($(SAMPLE_ENABLED),0)
+	@echo "Sample will be waived due to the above missing dependencies"
+else
+	@echo "Sample is ready - all dependencies have been met"
+endif
+
+cuSolverRf.o:cuSolverRf.cpp
+	$(EXEC) $(NVCC) $(INCLUDES) $(ALL_CCFLAGS) $(GENCODE_FLAGS) -o $@ -c $<
+
+mmio.c.o:mmio.c
+	$(EXEC) $(NVCC) $(INCLUDES) $(ALL_CCFLAGS) $(GENCODE_FLAGS) -o $@ -c $<
+
+mmio_wrapper.o:mmio_wrapper.cpp
+	$(EXEC) $(NVCC) $(INCLUDES) $(ALL_CCFLAGS) $(GENCODE_FLAGS) -o $@ -c $<
+
+cuSolverRf: cuSolverRf.o mmio.c.o mmio_wrapper.o
+	$(EXEC) $(NVCC) $(ALL_LDFLAGS) $(GENCODE_FLAGS) -o $@ $+ $(LIBRARIES)
+	$(EXEC) mkdir -p ../../../bin/$(TARGET_ARCH)/$(TARGET_OS)/$(BUILD_TYPE)
+	$(EXEC) cp $@ ../../../bin/$(TARGET_ARCH)/$(TARGET_OS)/$(BUILD_TYPE)
+
+run: build
+	$(EXEC) ./cuSolverRf
+
+testrun: build
+	$(EXEC) ./cuSolverRf -P=symrcm
+		$(EXEC) ./cuSolverRf -P=symamd
+
+clean:
+	rm -f cuSolverRf cuSolverRf.o mmio.c.o mmio_wrapper.o
+	rm -rf ../../../bin/$(TARGET_ARCH)/$(TARGET_OS)/$(BUILD_TYPE)/cuSolverRf
+
+clobber: clean

Samples/4_CUDA_Libraries/cuSolverRf/NsightEclipse.xml

@@ -0,0 +1,98 @@
+<?xml version="1.0" encoding="UTF-8"?> 
+<!DOCTYPE entry SYSTEM "SamplesInfo.dtd">
+<entry>
+  <name>cuSolverRf</name>
+  <cuda_api_list>
+    <driver>cuDoubleComplex</driver>
+    <driver>cuComplex</driver>
+    <driver>cuGet</driver>
+    <toolkit>cudaStreamDestroy</toolkit>
+    <toolkit>cudaFree</toolkit>
+    <toolkit>cudaStreamCreate</toolkit>
+    <toolkit>cudaDeviceSynchronize</toolkit>
+    <toolkit>cudaMalloc</toolkit>
+    <toolkit>cudaMemcpy</toolkit>
+  </cuda_api_list>
+  <description><![CDATA[A CUDA Sample that demonstrates cuSolver's refactorization library - CUSOLVERRF.]]></description>
+  <devicecompilation>whole</devicecompilation>
+  <files>
+    <file>lap2D_5pt_n100.mtx</file>
+    <file>lap3D_7pt_n20.mtx</file>
+  </files>
+  <includepaths>
+    <path>./</path>
+    <path>../</path>
+    <path>../../../Common</path>
+  </includepaths>
+  <keyconcepts>
+    <concept level="basic">Linear Algebra</concept>
+    <concept level="basic">CUSOLVER Library</concept>
+  </keyconcepts>
+  <keywords>
+    <keyword>CUSOLVER</keyword>
+    <keyword>Linear Algebra</keyword>
+  </keywords>
+  <libraries>
+    <library>cusolver</library>
+    <library>cublas</library>
+    <library>cusparse</library>
+  </libraries>
+  <librarypaths>
+  </librarypaths>
+  <nsight_eclipse>true</nsight_eclipse>
+  <primary_file>cuSolverRf.cpp</primary_file>
+  <qatests>
+    <qatest>-P=symrcm</qatest>
+    <qatest>-P=symamd</qatest>
+  </qatests>
+  <required_dependencies>
+    <dependency>CUSOLVER</dependency>
+    <dependency>CUBLAS</dependency>
+    <dependency>CUSPARSE</dependency>
+  </required_dependencies>
+  <scopes>
+    <scope>1:CUDA Basic Topics</scope>
+    <scope>3:Linear Algebra</scope>
+  </scopes>
+  <sm-arch>sm35</sm-arch>
+  <sm-arch>sm37</sm-arch>
+  <sm-arch>sm50</sm-arch>
+  <sm-arch>sm52</sm-arch>
+  <sm-arch>sm53</sm-arch>
+  <sm-arch>sm60</sm-arch>
+  <sm-arch>sm61</sm-arch>
+  <sm-arch>sm70</sm-arch>
+  <sm-arch>sm72</sm-arch>
+  <sm-arch>sm75</sm-arch>
+  <sm-arch>sm80</sm-arch>
+  <sm-arch>sm86</sm-arch>
+  <sm-arch>sm87</sm-arch>
+  <supported_envs>
+    <env>
+      <arch>x86_64</arch>
+      <platform>linux</platform>
+    </env>
+    <env>
+      <platform>windows7</platform>
+    </env>
+    <env>
+      <arch>x86_64</arch>
+      <platform>macosx</platform>
+    </env>
+    <env>
+      <arch>ppc64le</arch>
+      <platform>linux</platform>
+    </env>
+    <env>
+      <arch>arm</arch>
+    </env>
+    <env>
+      <arch>sbsa</arch>
+    </env>
+  </supported_envs>
+  <supported_sm_architectures>
+    <include>all</include>
+  </supported_sm_architectures>
+  <title>cuSolverRf Refactorization</title>
+  <type>exe</type>
+</entry>

Samples/4_CUDA_Libraries/cuSolverRf/README.md

@@ -0,0 +1,77 @@
+# cuSolverRf - cuSolverRf Refactorization
+
+## Description
+
+A CUDA Sample that demonstrates cuSolver's refactorization library - CUSOLVERRF.
+
+## Key Concepts
+
+Linear Algebra, CUSOLVER Library
+
+## Supported SM Architectures
+
+[SM 3.5 ](https://developer.nvidia.com/cuda-gpus)  [SM 3.7 ](https://developer.nvidia.com/cuda-gpus)  [SM 5.0 ](https://developer.nvidia.com/cuda-gpus)  [SM 5.2 ](https://developer.nvidia.com/cuda-gpus)  [SM 5.3 ](https://developer.nvidia.com/cuda-gpus)  [SM 6.0 ](https://developer.nvidia.com/cuda-gpus)  [SM 6.1 ](https://developer.nvidia.com/cuda-gpus)  [SM 7.0 ](https://developer.nvidia.com/cuda-gpus)  [SM 7.2 ](https://developer.nvidia.com/cuda-gpus)  [SM 7.5 ](https://developer.nvidia.com/cuda-gpus)  [SM 8.0 ](https://developer.nvidia.com/cuda-gpus)  [SM 8.6 ](https://developer.nvidia.com/cuda-gpus)  [SM 8.7 ](https://developer.nvidia.com/cuda-gpus)
+
+## Supported OSes
+
+Linux, Windows
+
+## Supported CPU Architecture
+
+x86_64, ppc64le, armv7l
+
+## CUDA APIs involved
+
+### [CUDA Driver API](http://docs.nvidia.com/cuda/cuda-driver-api/index.html)
+cuDoubleComplex, cuComplex, cuGet
+
+### [CUDA Runtime API](http://docs.nvidia.com/cuda/cuda-runtime-api/index.html)
+cudaStreamDestroy, cudaFree, cudaStreamCreate, cudaDeviceSynchronize, cudaMalloc, cudaMemcpy
+
+## Dependencies needed to build/run
+[CUSOLVER](../../README.md#cusolver), [CUBLAS](../../README.md#cublas), [CUSPARSE](../../README.md#cusparse)
+
+## Prerequisites
+
+Download and install the [CUDA Toolkit 11.6](https://developer.nvidia.com/cuda-downloads) for your corresponding platform.
+Make sure the dependencies mentioned in [Dependencies]() section above are installed.
+
+## Build and Run
+
+### Windows
+The Windows samples are built using the Visual Studio IDE. Solution files (.sln) are provided for each supported version of Visual Studio, using the format:
+```
+*_vs<version>.sln - for Visual Studio <version>
+```
+Each individual sample has its own set of solution files in its directory:
+
+To build/examine all the samples at once, the complete solution files should be used. To build/examine a single sample, the individual sample solution files should be used.
+> **Note:** Some samples require that the Microsoft DirectX SDK (June 2010 or newer) be installed and that the VC++ directory paths are properly set up (**Tools > Options...**). Check DirectX Dependencies section for details."
+
+### Linux
+The Linux samples are built using makefiles. To use the makefiles, change the current directory to the sample directory you wish to build, and run make:
+```
+$ cd <sample_dir>
+$ make
+```
+The samples makefiles can take advantage of certain options:
+*  **TARGET_ARCH=<arch>** - cross-compile targeting a specific architecture. Allowed architectures are x86_64, ppc64le, armv7l.
+    By default, TARGET_ARCH is set to HOST_ARCH. On a x86_64 machine, not setting TARGET_ARCH is the equivalent of setting TARGET_ARCH=x86_64.<br/>
+`$ make TARGET_ARCH=x86_64` <br/> `$ make TARGET_ARCH=ppc64le` <br/> `$ make TARGET_ARCH=armv7l` <br/>
+    See [here](http://docs.nvidia.com/cuda/cuda-samples/index.html#cross-samples) for more details.
+*   **dbg=1** - build with debug symbols
+    ```
+    $ make dbg=1
+    ```
+*   **SMS="A B ..."** - override the SM architectures for which the sample will be built, where `"A B ..."` is a space-delimited list of SM architectures. For example, to generate SASS for SM 50 and SM 60, use `SMS="50 60"`.
+    ```
+    $ make SMS="50 60"
+    ```
+
+*  **HOST_COMPILER=<host_compiler>** - override the default g++ host compiler. See the [Linux Installation Guide](http://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html#system-requirements) for a list of supported host compilers.
+```
+    $ make HOST_COMPILER=g++
+```
+
+## References (for more details)
+

Samples/4_CUDA_Libraries/cuSolverRf/cuSolverRf.cpp

@@ -0,0 +1,847 @@
+/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ *  A framework of refactorization process.
+ *
+ *  step 1: compute P*A*Q = L*U by
+ *    - reordering and
+ *    - LU with partial pivoting in cusolverSp
+ *
+ *  step 2: set up cusolverRf by (P, Q, L, U)
+ *
+ *  step 3: analyze and refactor A
+ *
+ *  How to use
+ *     ./cuSolverRf -P=symrcm -file <file>
+ *     ./cuSolverRf -P=symamd -file <file>
+ *
+ */
+
+#include "cusolverRf.h"
+
+#include <assert.h>
+#include <ctype.h>
+#include <cuda_runtime.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "cusolverSp.h"
+#include "cusolverSp_LOWLEVEL_PREVIEW.h"
+#include "helper_cuda.h"
+#include "helper_cusolver.h"
+#include "helper_string.h"
+
+template <typename T_ELEM>
+int loadMMSparseMatrix(char *filename, char elem_type, bool csrFormat, int *m,
+                       int *n, int *nnz, T_ELEM **aVal, int **aRowInd,
+                       int **aColInd, int extendSymMatrix);
+
+void UsageRF(void) {
+  printf("<options>\n");
+  printf("-h          : display this help\n");
+  printf("-P=<name>    : choose a reordering\n");
+  printf("              symrcm (Reverse Cuthill-McKee)\n");
+  printf("              symamd (Approximate Minimum Degree)\n");
+  printf("-file=<filename> : filename containing a matrix in MM format\n");
+  printf("-device=<device_id> : <device_id> if want to run on specific GPU\n");
+
+  exit(0);
+}
+
+void parseCommandLineArguments(int argc, char *argv[], struct testOpts &opts) {
+  memset(&opts, 0, sizeof(opts));
+
+  if (checkCmdLineFlag(argc, (const char **)argv, "-h")) {
+    UsageRF();
+  }
+
+  if (checkCmdLineFlag(argc, (const char **)argv, "P")) {
+    char *reorderType = NULL;
+    getCmdLineArgumentString(argc, (const char **)argv, "P", &reorderType);
+
+    if (reorderType) {
+      if ((STRCASECMP(reorderType, "symrcm") != 0) &&
+          (STRCASECMP(reorderType, "symamd") != 0)) {
+        printf("\nIncorrect argument passed to -P option\n");
+        UsageRF();
+      } else {
+        opts.reorder = reorderType;
+      }
+    }
+  }
+
+  if (!opts.reorder) {
+    opts.reorder = "symrcm";  // Setting default reordering to be symrcm.
+  }
+
+  if (checkCmdLineFlag(argc, (const char **)argv, "file")) {
+    char *fileName = 0;
+    getCmdLineArgumentString(argc, (const char **)argv, "file", &fileName);
+
+    if (fileName) {
+      opts.sparse_mat_filename = fileName;
+    } else {
+      printf("\nIncorrect filename passed to -file \n ");
+      UsageRF();
+    }
+  }
+}
+
+int main(int argc, char *argv[]) {
+  struct testOpts opts;
+  cusolverRfHandle_t cusolverRfH = NULL;  // refactorization
+  cusolverSpHandle_t cusolverSpH =
+      NULL;  // reordering, permutation and 1st LU factorization
+  cusparseHandle_t cusparseH = NULL;  // residual evaluation
+  cudaStream_t stream = NULL;
+  cusparseMatDescr_t descrA = NULL;  // A is a base-0 general matrix
+
+  csrluInfoHost_t info =
+      NULL;  // opaque info structure for LU with parital pivoting
+
+  int rowsA = 0;  // number of rows of A
+  int colsA = 0;  // number of columns of A
+  int nnzA = 0;   // number of nonzeros of A
+  int baseA = 0;  // base index in CSR format
+                  // cusolverRf only works for base-0
+
+  // cusolverRf only works for square matrix,
+  // assume n = rowsA = colsA
+
+  // CSR(A) from I/O
+  int *h_csrRowPtrA = NULL;  // <int> n+1
+  int *h_csrColIndA = NULL;  // <int> nnzA
+  double *h_csrValA = NULL;  // <double> nnzA
+
+  int *h_Qreorder = NULL;  // <int> n
+                           // reorder to reduce zero fill-in
+                           // Qreorder = symrcm(A) or Qreroder = symamd(A)
+  // B = Q*A*Q^T
+  int *h_csrRowPtrB = NULL;  // <int> n+1
+  int *h_csrColIndB = NULL;  // <int> nnzA
+  double *h_csrValB = NULL;  // <double> nnzA
+  int *h_mapBfromA = NULL;   // <int> nnzA
+
+  double *h_x = NULL;  // <double> n,  x = A \ b
+  double *h_b = NULL;  // <double> n, b = ones(m,1)
+  double *h_r = NULL;  // <double> n, r = b - A*x
+
+  // solve B*(Qx) = Q*b
+  double *h_xhat = NULL;  // <double> n, Q*x_hat = x
+  double *h_bhat = NULL;  // <double> n, b_hat = Q*b
+
+  size_t size_perm = 0;
+  size_t size_internal = 0;
+  size_t size_lu = 0;       // size of working space for csrlu
+  void *buffer_cpu = NULL;  // working space for
+                            // - permutation: B = Q*A*Q^T
+                            // - LU with partial pivoting in cusolverSp
+
+  // cusolverSp computes LU with partial pivoting
+  //     Plu*B*Qlu^T = L*U
+  //   where B = Q*A*Q^T
+  //
+  // nnzL and nnzU are not known until factorization is done.
+  // However upper bound of L+U is known after symbolic analysis of LU.
+  int *h_Plu = NULL;  // <int> n
+  int *h_Qlu = NULL;  // <int> n
+
+  int nnzL = 0;
+  int *h_csrRowPtrL = NULL;  // <int> n+1
+  int *h_csrColIndL = NULL;  // <int> nnzL
+  double *h_csrValL = NULL;  // <double> nnzL
+
+  int nnzU = 0;
+  int *h_csrRowPtrU = NULL;  // <int> n+1
+  int *h_csrColIndU = NULL;  // <int> nnzU
+  double *h_csrValU = NULL;  // <double> nnzU
+
+  int *h_P = NULL;  // <int> n, P = Plu * Qreorder
+  int *h_Q = NULL;  // <int> n, Q = Qlu * Qreorder
+
+  int *d_csrRowPtrA = NULL;  // <int> n+1
+  int *d_csrColIndA = NULL;  // <int> nnzA
+  double *d_csrValA = NULL;  // <double> nnzA
+  double *d_x = NULL;        // <double> n, x = A \ b
+  double *d_b = NULL;        // <double> n, a copy of h_b
+  double *d_r = NULL;        // <double> n, r = b - A*x
+
+  int *d_P = NULL;  // <int> n, P*A*Q^T = L*U
+  int *d_Q = NULL;  // <int> n
+
+  double *d_T = NULL;  // working space in cusolverRfSolve
+                       // |d_T| = n * nrhs
+
+  // the constants used in residual evaluation, r = b - A*x
+  const double minus_one = -1.0;
+  const double one = 1.0;
+  // the constants used in cusolverRf
+  // nzero is the value below which zero pivot is flagged.
+  // nboost is the value which is substitured for zero pivot.
+  double nzero = 0.0;
+  double nboost = 0.0;
+  // the constant used in cusolverSp
+  // singularity is -1 if A is invertible under tol
+  // tol determines the condition of singularity
+  // pivot_threshold decides pivoting strategy
+  int singularity = 0;
+  const double tol = 1.e-14;
+  const double pivot_threshold = 1.0;
+  // the constants used in cusolverRf
+  const cusolverRfFactorization_t fact_alg =
+      CUSOLVERRF_FACTORIZATION_ALG0;  // default
+  const cusolverRfTriangularSolve_t solve_alg =
+      CUSOLVERRF_TRIANGULAR_SOLVE_ALG1;  // default
+
+  double x_inf = 0.0;  // |x|
+  double r_inf = 0.0;  // |r|
+  double A_inf = 0.0;  // |A|
+  int errors = 0;
+
+  double start, stop;
+  double time_reorder;
+  double time_perm;
+  double time_sp_analysis;
+  double time_sp_factor;
+  double time_sp_solve;
+  double time_sp_extract;
+  double time_rf_assemble;
+  double time_rf_reset;
+  double time_rf_refactor;
+  double time_rf_solve;
+
+  parseCommandLineArguments(argc, argv, opts);
+
+  printf("step 1.1: preparation\n");
+  printf("step 1.1: read matrix market format\n");
+
+  findCudaDevice(argc, (const char **)argv);
+
+  if (opts.sparse_mat_filename == NULL) {
+    opts.sparse_mat_filename = sdkFindFilePath("lap2D_5pt_n100.mtx", argv[0]);
+    printf("Using default input file [%s]\n", opts.sparse_mat_filename);
+  } else {
+    printf("Using input file [%s]\n", opts.sparse_mat_filename);
+  }
+
+  if (opts.sparse_mat_filename) {
+    if (loadMMSparseMatrix<double>(opts.sparse_mat_filename, 'd', true, &rowsA,
+                                   &colsA, &nnzA, &h_csrValA, &h_csrRowPtrA,
+                                   &h_csrColIndA, true)) {
+      return 1;
+    }
+    baseA = h_csrRowPtrA[0];  // baseA = {0,1}
+  }
+
+  if (rowsA != colsA) {
+    fprintf(stderr, "Error: only support square matrix\n");
+    return 1;
+  }
+
+  printf("WARNING: cusolverRf only works for base-0 \n");
+  if (baseA) {
+    for (int i = 0; i <= rowsA; i++) {
+      h_csrRowPtrA[i]--;
+    }
+    for (int i = 0; i < nnzA; i++) {
+      h_csrColIndA[i]--;
+    }
+    baseA = 0;
+  }
+
+  printf("sparse matrix A is %d x %d with %d nonzeros, base=%d\n", rowsA, colsA,
+         nnzA, baseA);
+
+  checkCudaErrors(cusolverSpCreate(&cusolverSpH));
+  checkCudaErrors(cusparseCreate(&cusparseH));
+  checkCudaErrors(cudaStreamCreate(&stream));
+
+  checkCudaErrors(cusolverSpSetStream(cusolverSpH, stream));
+  checkCudaErrors(cusparseSetStream(cusparseH, stream));
+
+  checkCudaErrors(cusparseCreateMatDescr(&descrA));
+  checkCudaErrors(cusparseSetMatType(descrA, CUSPARSE_MATRIX_TYPE_GENERAL));
+
+  if (baseA) {
+    checkCudaErrors(cusparseSetMatIndexBase(descrA, CUSPARSE_INDEX_BASE_ONE));
+  } else {
+    checkCudaErrors(cusparseSetMatIndexBase(descrA, CUSPARSE_INDEX_BASE_ZERO));
+  }
+
+  h_Qreorder = (int *)malloc(sizeof(int) * colsA);
+
+  h_csrRowPtrB = (int *)malloc(sizeof(int) * (rowsA + 1));
+  h_csrColIndB = (int *)malloc(sizeof(int) * nnzA);
+  h_csrValB = (double *)malloc(sizeof(double) * nnzA);
+  h_mapBfromA = (int *)malloc(sizeof(int) * nnzA);
+
+  h_x = (double *)malloc(sizeof(double) * colsA);
+  h_b = (double *)malloc(sizeof(double) * rowsA);
+  h_r = (double *)malloc(sizeof(double) * rowsA);
+  h_xhat = (double *)malloc(sizeof(double) * colsA);
+  h_bhat = (double *)malloc(sizeof(double) * rowsA);
+
+  assert(NULL != h_Qreorder);
+
+  assert(NULL != h_csrRowPtrB);
+  assert(NULL != h_csrColIndB);
+  assert(NULL != h_csrValB);
+  assert(NULL != h_mapBfromA);
+
+  assert(NULL != h_x);
+  assert(NULL != h_b);
+  assert(NULL != h_r);
+  assert(NULL != h_xhat);
+  assert(NULL != h_bhat);
+
+  checkCudaErrors(
+      cudaMalloc((void **)&d_csrRowPtrA, sizeof(int) * (rowsA + 1)));
+  checkCudaErrors(cudaMalloc((void **)&d_csrColIndA, sizeof(int) * nnzA));
+  checkCudaErrors(cudaMalloc((void **)&d_csrValA, sizeof(double) * nnzA));
+  checkCudaErrors(cudaMalloc((void **)&d_x, sizeof(double) * colsA));
+  checkCudaErrors(cudaMalloc((void **)&d_b, sizeof(double) * rowsA));
+  checkCudaErrors(cudaMalloc((void **)&d_r, sizeof(double) * rowsA));
+  checkCudaErrors(cudaMalloc((void **)&d_P, sizeof(int) * rowsA));
+  checkCudaErrors(cudaMalloc((void **)&d_Q, sizeof(int) * colsA));
+  checkCudaErrors(cudaMalloc((void **)&d_T, sizeof(double) * rowsA * 1));
+
+  printf("step 1.2: set right hand side vector (b) to 1\n");
+  for (int row = 0; row < rowsA; row++) {
+    h_b[row] = 1.0;
+  }
+
+  printf("step 2: reorder the matrix to reduce zero fill-in\n");
+  printf("        Q = symrcm(A) or Q = symamd(A) \n");
+  start = second();
+  start = second();
+
+  if (0 == strcmp(opts.reorder, "symrcm")) {
+    checkCudaErrors(cusolverSpXcsrsymrcmHost(cusolverSpH, rowsA, nnzA, descrA,
+                                             h_csrRowPtrA, h_csrColIndA,
+                                             h_Qreorder));
+  } else if (0 == strcmp(opts.reorder, "symamd")) {
+    checkCudaErrors(cusolverSpXcsrsymamdHost(cusolverSpH, rowsA, nnzA, descrA,
+                                             h_csrRowPtrA, h_csrColIndA,
+                                             h_Qreorder));
+  } else {
+    fprintf(stderr, "Error: %s is unknow reordering\n", opts.reorder);
+    return 1;
+  }
+
+  stop = second();
+  time_reorder = stop - start;
+
+  printf("step 3: B = Q*A*Q^T\n");
+  memcpy(h_csrRowPtrB, h_csrRowPtrA, sizeof(int) * (rowsA + 1));
+  memcpy(h_csrColIndB, h_csrColIndA, sizeof(int) * nnzA);
+
+  start = second();
+  start = second();
+
+  checkCudaErrors(cusolverSpXcsrperm_bufferSizeHost(
+      cusolverSpH, rowsA, colsA, nnzA, descrA, h_csrRowPtrB, h_csrColIndB,
+      h_Qreorder, h_Qreorder, &size_perm));
+
+  if (buffer_cpu) {
+    free(buffer_cpu);
+  }
+  buffer_cpu = (void *)malloc(sizeof(char) * size_perm);
+  assert(NULL != buffer_cpu);
+
+  // h_mapBfromA = Identity
+  for (int j = 0; j < nnzA; j++) {
+    h_mapBfromA[j] = j;
+  }
+  checkCudaErrors(cusolverSpXcsrpermHost(
+      cusolverSpH, rowsA, colsA, nnzA, descrA, h_csrRowPtrB, h_csrColIndB,
+      h_Qreorder, h_Qreorder, h_mapBfromA, buffer_cpu));
+
+  // B = A( mapBfromA )
+  for (int j = 0; j < nnzA; j++) {
+    h_csrValB[j] = h_csrValA[h_mapBfromA[j]];
+  }
+
+  stop = second();
+  time_perm = stop - start;
+
+  printf("step 4: solve A*x = b by LU(B) in cusolverSp\n");
+
+  printf("step 4.1: create opaque info structure\n");
+  checkCudaErrors(cusolverSpCreateCsrluInfoHost(&info));
+
+  printf(
+      "step 4.2: analyze LU(B) to know structure of Q and R, and upper bound "
+      "for nnz(L+U)\n");
+  start = second();
+  start = second();
+
+  checkCudaErrors(cusolverSpXcsrluAnalysisHost(
+      cusolverSpH, rowsA, nnzA, descrA, h_csrRowPtrB, h_csrColIndB, info));
+
+  stop = second();
+  time_sp_analysis = stop - start;
+
+  printf("step 4.3: workspace for LU(B)\n");
+  checkCudaErrors(cusolverSpDcsrluBufferInfoHost(
+      cusolverSpH, rowsA, nnzA, descrA, h_csrValB, h_csrRowPtrB, h_csrColIndB,
+      info, &size_internal, &size_lu));
+
+  if (buffer_cpu) {
+    free(buffer_cpu);
+  }
+  buffer_cpu = (void *)malloc(sizeof(char) * size_lu);
+  assert(NULL != buffer_cpu);
+
+  printf("step 4.4: compute Ppivot*B = L*U \n");
+  start = second();
+  start = second();
+
+  checkCudaErrors(cusolverSpDcsrluFactorHost(
+      cusolverSpH, rowsA, nnzA, descrA, h_csrValB, h_csrRowPtrB, h_csrColIndB,
+      info, pivot_threshold, buffer_cpu));
+
+  stop = second();
+  time_sp_factor = stop - start;
+
+  // TODO: check singularity by tol
+  printf("step 4.5: check if the matrix is singular \n");
+  checkCudaErrors(
+      cusolverSpDcsrluZeroPivotHost(cusolverSpH, info, tol, &singularity));
+
+  if (0 <= singularity) {
+    fprintf(stderr, "Error: A is not invertible, singularity=%d\n",
+            singularity);
+    return 1;
+  }
+
+  printf("step 4.6: solve A*x = b \n");
+  printf("    i.e.  solve B*(Qx) = Q*b \n");
+  start = second();
+  start = second();
+
+  // b_hat = Q*b
+  for (int j = 0; j < rowsA; j++) {
+    h_bhat[j] = h_b[h_Qreorder[j]];
+  }
+  // B*x_hat = b_hat
+  checkCudaErrors(cusolverSpDcsrluSolveHost(cusolverSpH, rowsA, h_bhat, h_xhat,
+                                            info, buffer_cpu));
+
+  // x = Q^T * x_hat
+  for (int j = 0; j < rowsA; j++) {
+    h_x[h_Qreorder[j]] = h_xhat[j];
+  }
+
+  stop = second();
+  time_sp_solve = stop - start;
+
+  printf("step 4.7: evaluate residual r = b - A*x (result on CPU)\n");
+  // use GPU gemv to compute r = b - A*x
+  checkCudaErrors(cudaMemcpy(d_csrRowPtrA, h_csrRowPtrA,
+                             sizeof(int) * (rowsA + 1),
+                             cudaMemcpyHostToDevice));
+  checkCudaErrors(cudaMemcpy(d_csrColIndA, h_csrColIndA, sizeof(int) * nnzA,
+                             cudaMemcpyHostToDevice));
+  checkCudaErrors(cudaMemcpy(d_csrValA, h_csrValA, sizeof(double) * nnzA,
+                             cudaMemcpyHostToDevice));
+
+  checkCudaErrors(
+      cudaMemcpy(d_r, h_b, sizeof(double) * rowsA, cudaMemcpyHostToDevice));
+  checkCudaErrors(
+      cudaMemcpy(d_x, h_x, sizeof(double) * colsA, cudaMemcpyHostToDevice));
+
+  /* Wrap raw data into cuSPARSE generic API objects */
+  cusparseSpMatDescr_t matA = NULL;
+  if (baseA) {
+    checkCudaErrors(cusparseCreateCsr(&matA, rowsA, colsA, nnzA, d_csrRowPtrA,
+                                      d_csrColIndA, d_csrValA,
+                                      CUSPARSE_INDEX_32I, CUSPARSE_INDEX_32I,
+                                      CUSPARSE_INDEX_BASE_ONE, CUDA_R_64F));
+  } else {
+    checkCudaErrors(cusparseCreateCsr(&matA, rowsA, colsA, nnzA, d_csrRowPtrA,
+                                      d_csrColIndA, d_csrValA,
+                                      CUSPARSE_INDEX_32I, CUSPARSE_INDEX_32I,
+                                      CUSPARSE_INDEX_BASE_ZERO, CUDA_R_64F));
+  }
+
+  cusparseDnVecDescr_t vecx = NULL;
+  checkCudaErrors(cusparseCreateDnVec(&vecx, colsA, d_x, CUDA_R_64F));
+  cusparseDnVecDescr_t vecAx = NULL;
+  checkCudaErrors(cusparseCreateDnVec(&vecAx, rowsA, d_r, CUDA_R_64F));
+
+  /* Allocate workspace for cuSPARSE */
+  size_t bufferSize = 0;
+  checkCudaErrors(cusparseSpMV_bufferSize(
+      cusparseH, CUSPARSE_OPERATION_NON_TRANSPOSE, &minus_one, matA, vecx, &one,
+      vecAx, CUDA_R_64F, CUSPARSE_SPMV_ALG_DEFAULT, &bufferSize));
+  void *buffer = NULL;
+  checkCudaErrors(cudaMalloc(&buffer, bufferSize));
+
+  checkCudaErrors(cusparseSpMV(cusparseH, CUSPARSE_OPERATION_NON_TRANSPOSE,
+                               &minus_one, matA, vecx, &one, vecAx, CUDA_R_64F,
+                               CUSPARSE_SPMV_ALG_DEFAULT, buffer));
+
+  checkCudaErrors(
+      cudaMemcpy(h_r, d_r, sizeof(double) * rowsA, cudaMemcpyDeviceToHost));
+
+  x_inf = vec_norminf(colsA, h_x);
+  r_inf = vec_norminf(rowsA, h_r);
+  A_inf = csr_mat_norminf(rowsA, colsA, nnzA, descrA, h_csrValA, h_csrRowPtrA,
+                          h_csrColIndA);
+
+  printf("(CPU) |b - A*x| = %E \n", r_inf);
+  printf("(CPU) |A| = %E \n", A_inf);
+  printf("(CPU) |x| = %E \n", x_inf);
+  printf("(CPU) |b - A*x|/(|A|*|x|) = %E \n", r_inf / (A_inf * x_inf));
+
+  printf("step 5: extract P, Q, L and U from P*B*Q^T = L*U \n");
+  printf("        L has implicit unit diagonal\n");
+  start = second();
+  start = second();
+
+  checkCudaErrors(cusolverSpXcsrluNnzHost(cusolverSpH, &nnzL, &nnzU, info));
+
+  h_Plu = (int *)malloc(sizeof(int) * rowsA);
+  h_Qlu = (int *)malloc(sizeof(int) * colsA);
+
+  h_csrValL = (double *)malloc(sizeof(double) * nnzL);
+  h_csrRowPtrL = (int *)malloc(sizeof(int) * (rowsA + 1));
+  h_csrColIndL = (int *)malloc(sizeof(int) * nnzL);
+
+  h_csrValU = (double *)malloc(sizeof(double) * nnzU);
+  h_csrRowPtrU = (int *)malloc(sizeof(int) * (rowsA + 1));
+  h_csrColIndU = (int *)malloc(sizeof(int) * nnzU);
+
+  assert(NULL != h_Plu);
+  assert(NULL != h_Qlu);
+
+  assert(NULL != h_csrValL);
+  assert(NULL != h_csrRowPtrL);
+  assert(NULL != h_csrColIndL);
+
+  assert(NULL != h_csrValU);
+  assert(NULL != h_csrRowPtrU);
+  assert(NULL != h_csrColIndU);
+
+  checkCudaErrors(cusolverSpDcsrluExtractHost(
+      cusolverSpH, h_Plu, h_Qlu, descrA, h_csrValL, h_csrRowPtrL, h_csrColIndL,
+      descrA, h_csrValU, h_csrRowPtrU, h_csrColIndU, info, buffer_cpu));
+
+  stop = second();
+  time_sp_extract = stop - start;
+
+  printf("nnzL = %d, nnzU = %d\n", nnzL, nnzU);
+
+  /*  B = Qreorder*A*Qreorder^T
+   *  Plu*B*Qlu^T = L*U
+   *
+   *  (Plu*Qreorder)*A*(Qlu*Qreorder)^T = L*U
+   *
+   *  Let P = Plu*Qreroder, Q = Qlu*Qreorder,
+   *  then we have
+   *      P*A*Q^T = L*U
+   *  which is the fundamental relation in cusolverRf.
+   */
+  printf("step 6: form P*A*Q^T = L*U\n");
+
+  h_P = (int *)malloc(sizeof(int) * rowsA);
+  h_Q = (int *)malloc(sizeof(int) * colsA);
+  assert(NULL != h_P);
+  assert(NULL != h_Q);
+
+  printf("step 6.1: P = Plu*Qreroder\n");
+  // gather operation, P = Qreorder(Plu)
+  for (int j = 0; j < rowsA; j++) {
+    h_P[j] = h_Qreorder[h_Plu[j]];
+  }
+
+  printf("step 6.2: Q = Qlu*Qreorder \n");
+  // gather operation, Q = Qreorder(Qlu)
+  for (int j = 0; j < colsA; j++) {
+    h_Q[j] = h_Qreorder[h_Qlu[j]];
+  }
+
+  printf("step 7: create cusolverRf handle\n");
+  checkCudaErrors(cusolverRfCreate(&cusolverRfH));
+
+  printf("step 8: set parameters for cusolverRf \n");
+  // numerical values for checking "zeros" and for boosting.
+  checkCudaErrors(cusolverRfSetNumericProperties(cusolverRfH, nzero, nboost));
+
+  // choose algorithm for refactorization and solve
+  checkCudaErrors(cusolverRfSetAlgs(cusolverRfH, fact_alg, solve_alg));
+
+  // matrix mode: L and U are CSR format, and L has implicit unit diagonal
+  checkCudaErrors(
+      cusolverRfSetMatrixFormat(cusolverRfH, CUSOLVERRF_MATRIX_FORMAT_CSR,
+                                CUSOLVERRF_UNIT_DIAGONAL_ASSUMED_L));
+
+  // fast mode for matrix assembling
+  checkCudaErrors(cusolverRfSetResetValuesFastMode(
+      cusolverRfH, CUSOLVERRF_RESET_VALUES_FAST_MODE_ON));
+
+  printf("step 9: assemble P*A*Q = L*U \n");
+  start = second();
+  start = second();
+
+  checkCudaErrors(cusolverRfSetupHost(
+      rowsA, nnzA, h_csrRowPtrA, h_csrColIndA, h_csrValA, nnzL, h_csrRowPtrL,
+      h_csrColIndL, h_csrValL, nnzU, h_csrRowPtrU, h_csrColIndU, h_csrValU, h_P,
+      h_Q, cusolverRfH));
+
+  checkCudaErrors(cudaDeviceSynchronize());
+  stop = second();
+  time_rf_assemble = stop - start;
+
+  printf("step 10: analyze to extract parallelism \n");
+  checkCudaErrors(cusolverRfAnalyze(cusolverRfH));
+
+  printf("step 11: import A to cusolverRf \n");
+  checkCudaErrors(cudaMemcpy(d_csrRowPtrA, h_csrRowPtrA,
+                             sizeof(int) * (rowsA + 1),
+                             cudaMemcpyHostToDevice));
+  checkCudaErrors(cudaMemcpy(d_csrColIndA, h_csrColIndA, sizeof(int) * nnzA,
+                             cudaMemcpyHostToDevice));
+  checkCudaErrors(cudaMemcpy(d_csrValA, h_csrValA, sizeof(double) * nnzA,
+                             cudaMemcpyHostToDevice));
+  checkCudaErrors(
+      cudaMemcpy(d_P, h_P, sizeof(int) * rowsA, cudaMemcpyHostToDevice));
+  checkCudaErrors(
+      cudaMemcpy(d_Q, h_Q, sizeof(int) * colsA, cudaMemcpyHostToDevice));
+
+  start = second();
+  start = second();
+
+  checkCudaErrors(cusolverRfResetValues(rowsA, nnzA, d_csrRowPtrA, d_csrColIndA,
+                                        d_csrValA, d_P, d_Q, cusolverRfH));
+
+  checkCudaErrors(cudaDeviceSynchronize());
+  stop = second();
+  time_rf_reset = stop - start;
+
+  printf("step 12: refactorization \n");
+  start = second();
+  start = second();
+
+  checkCudaErrors(cusolverRfRefactor(cusolverRfH));
+
+  checkCudaErrors(cudaDeviceSynchronize());
+  stop = second();
+  time_rf_refactor = stop - start;
+
+  printf("step 13: solve A*x = b \n");
+  checkCudaErrors(
+      cudaMemcpy(d_x, h_b, sizeof(double) * rowsA, cudaMemcpyHostToDevice));
+
+  start = second();
+  start = second();
+
+  checkCudaErrors(
+      cusolverRfSolve(cusolverRfH, d_P, d_Q, 1, d_T, rowsA, d_x, rowsA));
+
+  checkCudaErrors(cudaDeviceSynchronize());
+  stop = second();
+  time_rf_solve = stop - start;
+
+  printf("step 14: evaluate residual r = b - A*x (result on GPU)\n");
+  checkCudaErrors(
+      cudaMemcpy(d_r, h_b, sizeof(double) * rowsA, cudaMemcpyHostToDevice));
+
+  checkCudaErrors(cusparseSpMV(cusparseH, CUSPARSE_OPERATION_NON_TRANSPOSE,
+                               &minus_one, matA, vecx, &one, vecAx, CUDA_R_64F,
+                               CUSPARSE_SPMV_ALG_DEFAULT, buffer));
+
+  checkCudaErrors(
+      cudaMemcpy(h_x, d_x, sizeof(double) * colsA, cudaMemcpyDeviceToHost));
+  checkCudaErrors(
+      cudaMemcpy(h_r, d_r, sizeof(double) * rowsA, cudaMemcpyDeviceToHost));
+
+  x_inf = vec_norminf(colsA, h_x);
+  r_inf = vec_norminf(rowsA, h_r);
+  printf("(GPU) |b - A*x| = %E \n", r_inf);
+  printf("(GPU) |A| = %E \n", A_inf);
+  printf("(GPU) |x| = %E \n", x_inf);
+  printf("(GPU) |b - A*x|/(|A|*|x|) = %E \n", r_inf / (A_inf * x_inf));
+
+  printf("===== statistics \n");
+  printf(" nnz(A) = %d, nnz(L+U) = %d, zero fill-in ratio = %f\n", nnzA,
+         nnzL + nnzU, ((double)(nnzL + nnzU)) / (double)nnzA);
+  printf("\n");
+  printf("===== timing profile \n");
+  printf(" reorder A   : %f sec\n", time_reorder);
+  printf(" B = Q*A*Q^T : %f sec\n", time_perm);
+  printf("\n");
+  printf(" cusolverSp LU analysis: %f sec\n", time_sp_analysis);
+  printf(" cusolverSp LU factor  : %f sec\n", time_sp_factor);
+  printf(" cusolverSp LU solve   : %f sec\n", time_sp_solve);
+  printf(" cusolverSp LU extract : %f sec\n", time_sp_extract);
+  printf("\n");
+  printf(" cusolverRf assemble : %f sec\n", time_rf_assemble);
+  printf(" cusolverRf reset    : %f sec\n", time_rf_reset);
+  printf(" cusolverRf refactor : %f sec\n", time_rf_refactor);
+  printf(" cusolverRf solve    : %f sec\n", time_rf_solve);
+
+  if (cusolverRfH) {
+    checkCudaErrors(cusolverRfDestroy(cusolverRfH));
+  }
+  if (cusolverSpH) {
+    checkCudaErrors(cusolverSpDestroy(cusolverSpH));
+  }
+  if (cusparseH) {
+    checkCudaErrors(cusparseDestroy(cusparseH));
+  }
+  if (stream) {
+    checkCudaErrors(cudaStreamDestroy(stream));
+  }
+  if (descrA) {
+    checkCudaErrors(cusparseDestroyMatDescr(descrA));
+  }
+  if (info) {
+    checkCudaErrors(cusolverSpDestroyCsrluInfoHost(info));
+  }
+
+  if (matA) {
+    checkCudaErrors(cusparseDestroySpMat(matA));
+  }
+  if (vecx) {
+    checkCudaErrors(cusparseDestroyDnVec(vecx));
+  }
+  if (vecAx) {
+    checkCudaErrors(cusparseDestroyDnVec(vecAx));
+  }
+
+  if (h_csrValA) {
+    free(h_csrValA);
+  }
+  if (h_csrRowPtrA) {
+    free(h_csrRowPtrA);
+  }
+  if (h_csrColIndA) {
+    free(h_csrColIndA);
+  }
+
+  if (h_Qreorder) {
+    free(h_Qreorder);
+  }
+
+  if (h_csrRowPtrB) {
+    free(h_csrRowPtrB);
+  }
+  if (h_csrColIndB) {
+    free(h_csrColIndB);
+  }
+  if (h_csrValB) {
+    free(h_csrValB);
+  }
+  if (h_mapBfromA) {
+    free(h_mapBfromA);
+  }
+
+  if (h_x) {
+    free(h_x);
+  }
+  if (h_b) {
+    free(h_b);
+  }
+  if (h_r) {
+    free(h_r);
+  }
+  if (h_xhat) {
+    free(h_xhat);
+  }
+  if (h_bhat) {
+    free(h_bhat);
+  }
+
+  if (buffer_cpu) {
+    free(buffer_cpu);
+  }
+
+  if (h_Plu) {
+    free(h_Plu);
+  }
+  if (h_Qlu) {
+    free(h_Qlu);
+  }
+  if (h_csrRowPtrL) {
+    free(h_csrRowPtrL);
+  }
+  if (h_csrColIndL) {
+    free(h_csrColIndL);
+  }
+  if (h_csrValL) {
+    free(h_csrValL);
+  }
+  if (h_csrRowPtrU) {
+    free(h_csrRowPtrU);
+  }
+  if (h_csrColIndU) {
+    free(h_csrColIndU);
+  }
+  if (h_csrValU) {
+    free(h_csrValU);
+  }
+
+  if (h_P) {
+    free(h_P);
+  }
+  if (h_Q) {
+    free(h_Q);
+  }
+
+  if (d_csrValA) {
+    checkCudaErrors(cudaFree(d_csrValA));
+  }
+  if (d_csrRowPtrA) {
+    checkCudaErrors(cudaFree(d_csrRowPtrA));
+  }
+  if (d_csrColIndA) {
+    checkCudaErrors(cudaFree(d_csrColIndA));
+  }
+  if (d_x) {
+    checkCudaErrors(cudaFree(d_x));
+  }
+  if (d_b) {
+    checkCudaErrors(cudaFree(d_b));
+  }
+  if (d_r) {
+    checkCudaErrors(cudaFree(d_r));
+  }
+  if (d_P) {
+    checkCudaErrors(cudaFree(d_P));
+  }
+  if (d_Q) {
+    checkCudaErrors(cudaFree(d_Q));
+  }
+  if (d_T) {
+    checkCudaErrors(cudaFree(d_T));
+  }
+
+  return 0;
+}

Samples/4_CUDA_Libraries/cuSolverRf/cuSolverRf_vs2017.sln

@@ -0,0 +1,20 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio 2017
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "cuSolverRf", "cuSolverRf_vs2017.vcxproj", "{997E0757-EA74-4A4E-A0FC-47D8C8831A15}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|x64 = Debug|x64
+		Release|x64 = Release|x64
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Debug|x64.ActiveCfg = Debug|x64
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Debug|x64.Build.0 = Debug|x64
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Release|x64.ActiveCfg = Release|x64
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Release|x64.Build.0 = Release|x64
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+EndGlobal

Samples/4_CUDA_Libraries/cuSolverRf/cuSolverRf_vs2017.vcxproj

@@ -0,0 +1,114 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <CUDAPropsPath Condition="'$(CUDAPropsPath)'==''">$(VCTargetsPath)\BuildCustomizations</CUDAPropsPath>
+  </PropertyGroup>
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Debug|x64">
+      <Configuration>Debug</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|x64">
+      <Configuration>Release</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>{997E0757-EA74-4A4E-A0FC-47D8C8831A15}</ProjectGuid>
+    <RootNamespace>cuSolverRf_vs2017</RootNamespace>
+    <ProjectName>cuSolverRf</ProjectName>
+    <CudaToolkitCustomDir />
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(WindowsTargetPlatformVersion)'==''">
+    <LatestTargetPlatformVersion>$([Microsoft.Build.Utilities.ToolLocationHelper]::GetLatestSDKTargetPlatformVersion('Windows', '10.0'))</LatestTargetPlatformVersion>
+    <WindowsTargetPlatformVersion Condition="'$(WindowsTargetPlatformVersion)' == ''">$(LatestTargetPlatformVersion)</WindowsTargetPlatformVersion>
+    <TargetPlatformVersion>$(WindowsTargetPlatformVersion)</TargetPlatformVersion>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <PropertyGroup>
+    <ConfigurationType>Application</ConfigurationType>
+    <CharacterSet>MultiByte</CharacterSet>
+    <PlatformToolset>v141</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)'=='Debug'">
+    <UseDebugLibraries>true</UseDebugLibraries>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)'=='Release'">
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <ImportGroup Label="ExtensionSettings">
+    <Import Project="$(CUDAPropsPath)\CUDA 11.6.props" />
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets">
+    <Import Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" />
+  </ImportGroup>
+  <PropertyGroup Label="UserMacros" />
+  <PropertyGroup>
+    <IntDir>$(Platform)/$(Configuration)/</IntDir>
+    <IncludePath>$(IncludePath)</IncludePath>
+    <CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRules />
+    <CodeAnalysisRuleAssemblies />
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Platform)'=='x64'">
+    <OutDir>../../../bin/win64/$(Configuration)/</OutDir>
+  </PropertyGroup>
+  <ItemDefinitionGroup>
+    <ClCompile>
+      <WarningLevel>Level3</WarningLevel>
+      <PreprocessorDefinitions>WIN32;_MBCS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <AdditionalIncludeDirectories>./;$(CudaToolkitDir)/include;../../../Common;$(CudaToolkitIncludeDir);</AdditionalIncludeDirectories>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <AdditionalDependencies>cusolver.lib;cublas.lib;cusparse.lib;cudart_static.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalLibraryDirectories>$(CudaToolkitLibDir);</AdditionalLibraryDirectories>
+      <OutputFile>$(OutDir)/cuSolverRf.exe</OutputFile>
+    </Link>
+    <CudaCompile>
+      <CodeGeneration>compute_35,sm_35;compute_37,sm_37;compute_50,sm_50;compute_52,sm_52;compute_60,sm_60;compute_61,sm_61;compute_70,sm_70;compute_75,sm_75;compute_80,sm_80;compute_86,sm_86;</CodeGeneration>
+      <AdditionalOptions>-Xcompiler "/wd 4819"  --threads 0 </AdditionalOptions>
+      <Include>./;../../../Common</Include>
+      <Defines>WIN32</Defines>
+    </CudaCompile>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)'=='Debug'">
+    <ClCompile>
+      <Optimization>Disabled</Optimization>
+      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <LinkTimeCodeGeneration>Default</LinkTimeCodeGeneration>
+    </Link>
+    <CudaCompile>
+      <Runtime>MTd</Runtime>
+      <TargetMachinePlatform>64</TargetMachinePlatform>
+    </CudaCompile>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)'=='Release'">
+    <ClCompile>
+      <Optimization>MaxSpeed</Optimization>
+      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <GenerateDebugInformation>false</GenerateDebugInformation>
+      <LinkTimeCodeGeneration>UseLinkTimeCodeGeneration</LinkTimeCodeGeneration>
+    </Link>
+    <CudaCompile>
+      <Runtime>MT</Runtime>
+      <TargetMachinePlatform>64</TargetMachinePlatform>
+    </CudaCompile>
+  </ItemDefinitionGroup>
+  <ItemGroup>
+    <ClCompile Include="cuSolverRf.cpp" />
+    <ClCompile Include="mmio.c" />
+    <ClCompile Include="mmio_wrapper.cpp" />
+    <ClInclude Include="mmio.h" />
+  </ItemGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <ImportGroup Label="ExtensionTargets">
+    <Import Project="$(CUDAPropsPath)\CUDA 11.6.targets" />
+  </ImportGroup>
+</Project>

Samples/4_CUDA_Libraries/cuSolverRf/cuSolverRf_vs2019.sln

@@ -0,0 +1,20 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio 2019
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "cuSolverRf", "cuSolverRf_vs2019.vcxproj", "{997E0757-EA74-4A4E-A0FC-47D8C8831A15}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|x64 = Debug|x64
+		Release|x64 = Release|x64
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Debug|x64.ActiveCfg = Debug|x64
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Debug|x64.Build.0 = Debug|x64
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Release|x64.ActiveCfg = Release|x64
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Release|x64.Build.0 = Release|x64
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+EndGlobal

Samples/4_CUDA_Libraries/cuSolverRf/cuSolverRf_vs2019.vcxproj

@@ -0,0 +1,110 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <CUDAPropsPath Condition="'$(CUDAPropsPath)'==''">$(VCTargetsPath)\BuildCustomizations</CUDAPropsPath>
+  </PropertyGroup>
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Debug|x64">
+      <Configuration>Debug</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|x64">
+      <Configuration>Release</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>{997E0757-EA74-4A4E-A0FC-47D8C8831A15}</ProjectGuid>
+    <RootNamespace>cuSolverRf_vs2019</RootNamespace>
+    <ProjectName>cuSolverRf</ProjectName>
+    <CudaToolkitCustomDir />
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <PropertyGroup>
+    <ConfigurationType>Application</ConfigurationType>
+    <CharacterSet>MultiByte</CharacterSet>
+    <PlatformToolset>v142</PlatformToolset>
+    <WindowsTargetPlatformVersion>10.0</WindowsTargetPlatformVersion>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)'=='Debug'">
+    <UseDebugLibraries>true</UseDebugLibraries>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)'=='Release'">
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <ImportGroup Label="ExtensionSettings">
+    <Import Project="$(CUDAPropsPath)\CUDA 11.6.props" />
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets">
+    <Import Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" />
+  </ImportGroup>
+  <PropertyGroup Label="UserMacros" />
+  <PropertyGroup>
+    <IntDir>$(Platform)/$(Configuration)/</IntDir>
+    <IncludePath>$(IncludePath)</IncludePath>
+    <CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRules />
+    <CodeAnalysisRuleAssemblies />
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Platform)'=='x64'">
+    <OutDir>../../../bin/win64/$(Configuration)/</OutDir>
+  </PropertyGroup>
+  <ItemDefinitionGroup>
+    <ClCompile>
+      <WarningLevel>Level3</WarningLevel>
+      <PreprocessorDefinitions>WIN32;_MBCS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <AdditionalIncludeDirectories>./;$(CudaToolkitDir)/include;../../../Common;$(CudaToolkitIncludeDir);</AdditionalIncludeDirectories>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <AdditionalDependencies>cusolver.lib;cublas.lib;cusparse.lib;cudart_static.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalLibraryDirectories>$(CudaToolkitLibDir);</AdditionalLibraryDirectories>
+      <OutputFile>$(OutDir)/cuSolverRf.exe</OutputFile>
+    </Link>
+    <CudaCompile>
+      <CodeGeneration>compute_35,sm_35;compute_37,sm_37;compute_50,sm_50;compute_52,sm_52;compute_60,sm_60;compute_61,sm_61;compute_70,sm_70;compute_75,sm_75;compute_80,sm_80;compute_86,sm_86;</CodeGeneration>
+      <AdditionalOptions>-Xcompiler "/wd 4819"  --threads 0 </AdditionalOptions>
+      <Include>./;../../../Common</Include>
+      <Defines>WIN32</Defines>
+    </CudaCompile>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)'=='Debug'">
+    <ClCompile>
+      <Optimization>Disabled</Optimization>
+      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <LinkTimeCodeGeneration>Default</LinkTimeCodeGeneration>
+    </Link>
+    <CudaCompile>
+      <Runtime>MTd</Runtime>
+      <TargetMachinePlatform>64</TargetMachinePlatform>
+    </CudaCompile>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)'=='Release'">
+    <ClCompile>
+      <Optimization>MaxSpeed</Optimization>
+      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <GenerateDebugInformation>false</GenerateDebugInformation>
+      <LinkTimeCodeGeneration>UseLinkTimeCodeGeneration</LinkTimeCodeGeneration>
+    </Link>
+    <CudaCompile>
+      <Runtime>MT</Runtime>
+      <TargetMachinePlatform>64</TargetMachinePlatform>
+    </CudaCompile>
+  </ItemDefinitionGroup>
+  <ItemGroup>
+    <ClCompile Include="cuSolverRf.cpp" />
+    <ClCompile Include="mmio.c" />
+    <ClCompile Include="mmio_wrapper.cpp" />
+    <ClInclude Include="mmio.h" />
+  </ItemGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <ImportGroup Label="ExtensionTargets">
+    <Import Project="$(CUDAPropsPath)\CUDA 11.6.targets" />
+  </ImportGroup>
+</Project>

Samples/4_CUDA_Libraries/cuSolverRf/cuSolverRf_vs2022.sln

@@ -0,0 +1,20 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio 2022
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "cuSolverRf", "cuSolverRf_vs2022.vcxproj", "{997E0757-EA74-4A4E-A0FC-47D8C8831A15}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|x64 = Debug|x64
+		Release|x64 = Release|x64
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Debug|x64.ActiveCfg = Debug|x64
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Debug|x64.Build.0 = Debug|x64
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Release|x64.ActiveCfg = Release|x64
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Release|x64.Build.0 = Release|x64
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+EndGlobal

Samples/4_CUDA_Libraries/cuSolverRf/cuSolverRf_vs2022.vcxproj

@@ -0,0 +1,110 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <CUDAPropsPath Condition="'$(CUDAPropsPath)'==''">$(VCTargetsPath)\BuildCustomizations</CUDAPropsPath>
+  </PropertyGroup>
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Debug|x64">
+      <Configuration>Debug</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|x64">
+      <Configuration>Release</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>{997E0757-EA74-4A4E-A0FC-47D8C8831A15}</ProjectGuid>
+    <RootNamespace>cuSolverRf_vs2022</RootNamespace>
+    <ProjectName>cuSolverRf</ProjectName>
+    <CudaToolkitCustomDir />
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <PropertyGroup>
+    <ConfigurationType>Application</ConfigurationType>
+    <CharacterSet>MultiByte</CharacterSet>
+    <PlatformToolset>v143</PlatformToolset>
+    <WindowsTargetPlatformVersion>10.0</WindowsTargetPlatformVersion>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)'=='Debug'">
+    <UseDebugLibraries>true</UseDebugLibraries>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)'=='Release'">
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <ImportGroup Label="ExtensionSettings">
+    <Import Project="$(CUDAPropsPath)\CUDA 11.6.props" />
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets">
+    <Import Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" />
+  </ImportGroup>
+  <PropertyGroup Label="UserMacros" />
+  <PropertyGroup>
+    <IntDir>$(Platform)/$(Configuration)/</IntDir>
+    <IncludePath>$(IncludePath)</IncludePath>
+    <CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRules />
+    <CodeAnalysisRuleAssemblies />
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Platform)'=='x64'">
+    <OutDir>../../../bin/win64/$(Configuration)/</OutDir>
+  </PropertyGroup>
+  <ItemDefinitionGroup>
+    <ClCompile>
+      <WarningLevel>Level3</WarningLevel>
+      <PreprocessorDefinitions>WIN32;_MBCS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <AdditionalIncludeDirectories>./;$(CudaToolkitDir)/include;../../../Common;$(CudaToolkitIncludeDir);</AdditionalIncludeDirectories>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <AdditionalDependencies>cusolver.lib;cublas.lib;cusparse.lib;cudart_static.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalLibraryDirectories>$(CudaToolkitLibDir);</AdditionalLibraryDirectories>
+      <OutputFile>$(OutDir)/cuSolverRf.exe</OutputFile>
+    </Link>
+    <CudaCompile>
+      <CodeGeneration>compute_35,sm_35;compute_37,sm_37;compute_50,sm_50;compute_52,sm_52;compute_60,sm_60;compute_61,sm_61;compute_70,sm_70;compute_75,sm_75;compute_80,sm_80;compute_86,sm_86;</CodeGeneration>
+      <AdditionalOptions>-Xcompiler "/wd 4819"  --threads 0 </AdditionalOptions>
+      <Include>./;../../../Common</Include>
+      <Defines>WIN32</Defines>
+    </CudaCompile>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)'=='Debug'">
+    <ClCompile>
+      <Optimization>Disabled</Optimization>
+      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <LinkTimeCodeGeneration>Default</LinkTimeCodeGeneration>
+    </Link>
+    <CudaCompile>
+      <Runtime>MTd</Runtime>
+      <TargetMachinePlatform>64</TargetMachinePlatform>
+    </CudaCompile>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)'=='Release'">
+    <ClCompile>
+      <Optimization>MaxSpeed</Optimization>
+      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <GenerateDebugInformation>false</GenerateDebugInformation>
+      <LinkTimeCodeGeneration>UseLinkTimeCodeGeneration</LinkTimeCodeGeneration>
+    </Link>
+    <CudaCompile>
+      <Runtime>MT</Runtime>
+      <TargetMachinePlatform>64</TargetMachinePlatform>
+    </CudaCompile>
+  </ItemDefinitionGroup>
+  <ItemGroup>
+    <ClCompile Include="cuSolverRf.cpp" />
+    <ClCompile Include="mmio.c" />
+    <ClCompile Include="mmio_wrapper.cpp" />
+    <ClInclude Include="mmio.h" />
+  </ItemGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <ImportGroup Label="ExtensionTargets">
+    <Import Project="$(CUDAPropsPath)\CUDA 11.6.targets" />
+  </ImportGroup>
+</Project>

Samples/4_CUDA_Libraries/cuSolverRf/mmio.c

@@ -0,0 +1,515 @@
+/* 
+*   Matrix Market I/O library for ANSI C
+*
+*   See http://math.nist.gov/MatrixMarket for details.
+*
+*
+*/
+
+/* avoid Windows warnings (for example: strcpy, fscanf, etc.) */
+#if defined(_WIN32)  
+#define _CRT_SECURE_NO_WARNINGS
+#endif
+
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include <ctype.h>
+
+#include "mmio.h"
+
+int mm_read_unsymmetric_sparse(const char *fname, int *M_, int *N_, int *nz_,
+                double **val_, int **I_, int **J_)
+{
+    FILE *f;
+    MM_typecode matcode;
+    int M, N, nz;
+    int i;
+    double *val;
+    int *I, *J;
+ 
+    if ((f = fopen(fname, "r")) == NULL)
+            return -1;
+ 
+ 
+    if (mm_read_banner(f, &matcode) != 0)
+    {
+        printf("mm_read_unsymetric: Could not process Matrix Market banner ");
+        printf(" in file [%s]\n", fname);
+        return -1;
+    }
+ 
+ 
+ 
+    if ( !(mm_is_real(matcode) && mm_is_matrix(matcode) &&
+            mm_is_sparse(matcode)))
+    {
+        fprintf(stderr, "Sorry, this application does not support ");
+        fprintf(stderr, "Market Market type: [%s]\n",
+                mm_typecode_to_str(matcode));
+        return -1;
+    }
+ 
+    /* find out size of sparse matrix: M, N, nz .... */
+ 
+    if (mm_read_mtx_crd_size(f, &M, &N, &nz) !=0)
+    {
+        fprintf(stderr, "read_unsymmetric_sparse(): could not parse matrix size.\n");
+        return -1;
+    }
+ 
+    *M_ = M;
+    *N_ = N;
+    *nz_ = nz;
+ 
+    /* reserve memory for matrices */
+ 
+    I = (int *) malloc(nz * sizeof(int));
+    J = (int *) malloc(nz * sizeof(int));
+    val = (double *) malloc(nz * sizeof(double));
+ 
+    *val_ = val;
+    *I_ = I;
+    *J_ = J;
+ 
+    /* NOTE: when reading in doubles, ANSI C requires the use of the "l"  */
+    /*   specifier as in "%lg", "%lf", "%le", otherwise errors will occur */
+    /*  (ANSI C X3.159-1989, Sec. 4.9.6.2, p. 136 lines 13-15)            */
+ 
+    for (i=0; i<nz; i++)
+    {
+        if (fscanf(f, "%d %d %lg\n", &I[i], &J[i], &val[i]) != 3) {
+            return -1;
+        }
+        I[i]--;  /* adjust from 1-based to 0-based */
+        J[i]--;
+    }
+    fclose(f);
+ 
+    return 0;
+}
+
+int mm_is_valid(MM_typecode matcode)
+{
+    if (!mm_is_matrix(matcode)) return 0;
+    if (mm_is_dense(matcode) && mm_is_pattern(matcode)) return 0;
+    if (mm_is_real(matcode) && mm_is_hermitian(matcode)) return 0;
+    if (mm_is_pattern(matcode) && (mm_is_hermitian(matcode) ||
+                mm_is_skew(matcode))) return 0;
+    return 1;
+}
+
+int mm_read_banner(FILE *f, MM_typecode *matcode)
+{
+    char line[MM_MAX_LINE_LENGTH];
+    char banner[MM_MAX_TOKEN_LENGTH];
+    char mtx[MM_MAX_TOKEN_LENGTH];
+    char crd[MM_MAX_TOKEN_LENGTH];
+    char data_type[MM_MAX_TOKEN_LENGTH];
+    char storage_scheme[MM_MAX_TOKEN_LENGTH];
+    char *p;
+
+
+    mm_clear_typecode(matcode);
+
+    if (fgets(line, MM_MAX_LINE_LENGTH, f) == NULL)
+        return MM_PREMATURE_EOF;
+
+    if (sscanf(line, "%s %s %s %s %s", banner, mtx, crd, data_type,
+        storage_scheme) != 5)
+        return MM_PREMATURE_EOF;
+
+    for (p=mtx; *p!='\0'; *p=tolower(*p),p++);  /* convert to lower case */
+    for (p=crd; *p!='\0'; *p=tolower(*p),p++);
+    for (p=data_type; *p!='\0'; *p=tolower(*p),p++);
+    for (p=storage_scheme; *p!='\0'; *p=tolower(*p),p++);
+
+    /* check for banner */
+    if (strncmp(banner, MatrixMarketBanner, strlen(MatrixMarketBanner)) != 0)
+        return MM_NO_HEADER;
+
+    /* first field should be "mtx" */
+    if (strcmp(mtx, MM_MTX_STR) != 0)
+        return  MM_UNSUPPORTED_TYPE;
+    mm_set_matrix(matcode);
+
+
+    /* second field describes whether this is a sparse matrix (in coordinate
+            storgae) or a dense array */
+
+
+    if (strcmp(crd, MM_SPARSE_STR) == 0)
+        mm_set_sparse(matcode);
+    else
+    if (strcmp(crd, MM_DENSE_STR) == 0)
+            mm_set_dense(matcode);
+    else
+        return MM_UNSUPPORTED_TYPE;
+
+    /* third field */
+
+    if (strcmp(data_type, MM_REAL_STR) == 0)
+        mm_set_real(matcode);
+    else
+    if (strcmp(data_type, MM_COMPLEX_STR) == 0)
+        mm_set_complex(matcode);
+    else
+    if (strcmp(data_type, MM_PATTERN_STR) == 0)
+        mm_set_pattern(matcode);
+    else
+    if (strcmp(data_type, MM_INT_STR) == 0)
+        mm_set_integer(matcode);
+    else
+        return MM_UNSUPPORTED_TYPE;
+
+    /* fourth field */
+
+    if (strcmp(storage_scheme, MM_GENERAL_STR) == 0)
+        mm_set_general(matcode);
+    else
+    if (strcmp(storage_scheme, MM_SYMM_STR) == 0)
+        mm_set_symmetric(matcode);
+    else
+    if (strcmp(storage_scheme, MM_HERM_STR) == 0)
+        mm_set_hermitian(matcode);
+    else
+    if (strcmp(storage_scheme, MM_SKEW_STR) == 0)
+        mm_set_skew(matcode);
+    else
+        return MM_UNSUPPORTED_TYPE;
+
+    return 0;
+}
+
+int mm_write_mtx_crd_size(FILE *f, int M, int N, int nz)
+{
+    if (fprintf(f, "%d %d %d\n", M, N, nz) != 3)
+        return MM_COULD_NOT_WRITE_FILE;
+    else
+        return 0;
+}
+
+int mm_read_mtx_crd_size(FILE *f, int *M, int *N, int *nz )
+{
+    char line[MM_MAX_LINE_LENGTH];
+    int num_items_read;
+
+    /* set return null parameter values, in case we exit with errors */
+    *M = *N = *nz = 0;
+
+    /* now continue scanning until you reach the end-of-comments */
+    do
+    {
+        if (fgets(line,MM_MAX_LINE_LENGTH,f) == NULL)
+            return MM_PREMATURE_EOF;
+    }while (line[0] == '%');
+
+    /* line[] is either blank or has M,N, nz */
+    if (sscanf(line, "%d %d %d", M, N, nz) == 3)
+        return 0;
+
+    else
+    do
+    {
+        num_items_read = fscanf(f, "%d %d %d", M, N, nz);
+        if (num_items_read == EOF) return MM_PREMATURE_EOF;
+    }
+    while (num_items_read != 3);
+
+    return 0;
+}
+
+
+int mm_read_mtx_array_size(FILE *f, int *M, int *N)
+{
+    char line[MM_MAX_LINE_LENGTH];
+    int num_items_read;
+    /* set return null parameter values, in case we exit with errors */
+    *M = *N = 0;
+
+    /* now continue scanning until you reach the end-of-comments */
+    do
+    {
+        if (fgets(line,MM_MAX_LINE_LENGTH,f) == NULL)
+            return MM_PREMATURE_EOF;
+    }while (line[0] == '%');
+
+    /* line[] is either blank or has M,N, nz */
+    if (sscanf(line, "%d %d", M, N) == 2)
+        return 0;
+
+    else /* we have a blank line */
+    do
+    {
+        num_items_read = fscanf(f, "%d %d", M, N);
+        if (num_items_read == EOF) return MM_PREMATURE_EOF;
+    }
+    while (num_items_read != 2);
+
+    return 0;
+}
+
+int mm_write_mtx_array_size(FILE *f, int M, int N)
+{
+    if (fprintf(f, "%d %d\n", M, N) != 2)
+        return MM_COULD_NOT_WRITE_FILE;
+    else
+        return 0;
+}
+
+
+
+/*-------------------------------------------------------------------------*/
+
+/******************************************************************/
+/* use when I[], J[], and val[]J, and val[] are already allocated */
+/******************************************************************/
+
+int mm_read_mtx_crd_data(FILE *f, int M, int N, int nz, int I[], int J[],
+        double val[], MM_typecode matcode)
+{
+    int i;
+    if (mm_is_complex(matcode))
+    {
+        for (i=0; i<nz; i++)
+            if (fscanf(f, "%d %d %lg %lg", &I[i], &J[i], &val[2*i], &val[2*i+1])
+                != 4) return MM_PREMATURE_EOF;
+    }
+    else if (mm_is_real(matcode) || mm_is_integer(matcode))
+    {
+        for (i=0; i<nz; i++)
+        {
+            if (fscanf(f, "%d %d %lg\n", &I[i], &J[i], &val[i])
+                != 3) return MM_PREMATURE_EOF;
+
+        }
+    }
+
+    else if (mm_is_pattern(matcode))
+    {
+        for (i=0; i<nz; i++)
+            if (fscanf(f, "%d %d", &I[i], &J[i])
+                != 2) return MM_PREMATURE_EOF;
+    }
+    else
+        return MM_UNSUPPORTED_TYPE;
+
+    return 0;
+}
+
+int mm_read_mtx_crd_entry(FILE *f, int *I, int *J,
+        double *real, double *imag, MM_typecode matcode)
+{
+    if (mm_is_complex(matcode))
+    {
+            if (fscanf(f, "%d %d %lg %lg", I, J, real, imag)
+                != 4) return MM_PREMATURE_EOF;
+    }
+    else if (mm_is_real(matcode) || mm_is_integer(matcode))
+    {
+            if (fscanf(f, "%d %d %lg\n", I, J, real)
+                != 3) return MM_PREMATURE_EOF;
+
+    }
+
+    else if (mm_is_pattern(matcode))
+    {
+            if (fscanf(f, "%d %d", I, J) != 2) return MM_PREMATURE_EOF;
+    }
+    else
+        return MM_UNSUPPORTED_TYPE;
+
+    return 0;
+}
+
+
+/************************************************************************
+    mm_read_mtx_crd()  fills M, N, nz, array of values, and return
+                        type code, e.g. 'MCRS'
+
+                        if matrix is complex, values[] is of size 2*nz,
+                            (nz pairs of real/imaginary values)
+************************************************************************/
+
+int mm_read_mtx_crd(char *fname, int *M, int *N, int *nz, int **I, int **J,
+        double **val, MM_typecode *matcode)
+{
+    int ret_code;
+    FILE *f;
+
+    if (strcmp(fname, "stdin") == 0) f=stdin;
+    else
+    if ((f = fopen(fname, "r")) == NULL)
+        return MM_COULD_NOT_READ_FILE;
+
+
+    if ((ret_code = mm_read_banner(f, matcode)) != 0)
+        return ret_code;
+
+    if (!(mm_is_valid(*matcode) && mm_is_sparse(*matcode) &&
+            mm_is_matrix(*matcode)))
+        return MM_UNSUPPORTED_TYPE;
+
+    if ((ret_code = mm_read_mtx_crd_size(f, M, N, nz)) != 0)
+        return ret_code;
+
+
+    *I = (int *)  malloc(*nz * sizeof(int));
+    *J = (int *)  malloc(*nz * sizeof(int));
+    *val = NULL;
+
+    if (mm_is_complex(*matcode))
+    {
+        *val = (double *) malloc(*nz * 2 * sizeof(double));
+        ret_code = mm_read_mtx_crd_data(f, *M, *N, *nz, *I, *J, *val, 
+                *matcode);
+        if (ret_code != 0) return ret_code;
+    }
+    else if (mm_is_real(*matcode) || mm_is_integer(*matcode))
+    {
+        *val = (double *) malloc(*nz * sizeof(double));
+        ret_code = mm_read_mtx_crd_data(f, *M, *N, *nz, *I, *J, *val, 
+                *matcode);
+        if (ret_code != 0) return ret_code;
+    }
+
+    else if (mm_is_pattern(*matcode))
+    {
+        ret_code = mm_read_mtx_crd_data(f, *M, *N, *nz, *I, *J, *val, 
+                *matcode);
+        if (ret_code != 0) return ret_code;
+    }
+
+    if (f != stdin) fclose(f);
+    return 0;
+}
+
+int mm_write_banner(FILE *f, MM_typecode matcode)
+{
+    char *str = mm_typecode_to_str(matcode);
+    int ret_code;
+
+    ret_code = fprintf(f, "%s %s\n", MatrixMarketBanner, str);
+    free(str);
+    if (ret_code !=2 )
+        return MM_COULD_NOT_WRITE_FILE;
+    else
+        return 0;
+}
+
+int mm_write_mtx_crd(char fname[], int M, int N, int nz, int I[], int J[],
+        double val[], MM_typecode matcode)
+{
+    FILE *f;
+    int i;
+
+    if (strcmp(fname, "stdout") == 0)
+        f = stdout;
+    else
+    if ((f = fopen(fname, "w")) == NULL)
+        return MM_COULD_NOT_WRITE_FILE;
+
+    /* print banner followed by typecode */
+    fprintf(f, "%s ", MatrixMarketBanner);
+    fprintf(f, "%s\n", mm_typecode_to_str(matcode));
+
+    /* print matrix sizes and nonzeros */
+    fprintf(f, "%d %d %d\n", M, N, nz);
+
+    /* print values */
+    if (mm_is_pattern(matcode))
+        for (i=0; i<nz; i++)
+            fprintf(f, "%d %d\n", I[i], J[i]);
+    else
+    if (mm_is_integer(matcode))
+        for (i=0; i<nz; i++)
+            fprintf(f, "%d %d %d\n", I[i], J[i], (int)val[i]);
+    else
+    if (mm_is_real(matcode))
+        for (i=0; i<nz; i++)
+            fprintf(f, "%d %d %20.16g\n", I[i], J[i], val[i]);
+    else
+    if (mm_is_complex(matcode))
+        for (i=0; i<nz; i++)
+            fprintf(f, "%d %d %20.16g %20.16g\n", I[i], J[i], val[2*i],
+                        val[2*i+1]);
+    else
+    {
+        if (f != stdout) fclose(f);
+        return MM_UNSUPPORTED_TYPE;
+    }
+
+    if (f !=stdout) fclose(f);
+
+    return 0;
+}
+
+/**
+*  Create a new copy of a string s.  mm_strdup() is a common routine, but
+*  not part of ANSI C, so it is included here.  Used by mm_typecode_to_str().
+*
+*/
+static char *mm_strdup(const char *s)
+{
+    size_t len = strlen(s);
+    char *s2 = (char *) malloc((len+1)*sizeof(char));
+    return strcpy(s2, s);
+}
+
+char  *mm_typecode_to_str(MM_typecode matcode)
+{
+    char buffer[MM_MAX_LINE_LENGTH];
+    char *types[4];
+    //char *mm_strdup(const char *);
+    //int error =0;
+
+    /* check for MTX type */
+    if (mm_is_matrix(matcode)) 
+        types[0] = MM_MTX_STR;
+    else
+        return NULL; //  error=1;
+
+    /* check for CRD or ARR matrix */
+    if (mm_is_sparse(matcode))
+        types[1] = MM_SPARSE_STR;
+    else
+    if (mm_is_dense(matcode))
+        types[1] = MM_DENSE_STR;
+    else
+        return NULL;
+
+    /* check for element data type */
+    if (mm_is_real(matcode))
+        types[2] = MM_REAL_STR;
+    else
+    if (mm_is_complex(matcode))
+        types[2] = MM_COMPLEX_STR;
+    else
+    if (mm_is_pattern(matcode))
+        types[2] = MM_PATTERN_STR;
+    else
+    if (mm_is_integer(matcode))
+        types[2] = MM_INT_STR;
+    else
+        return NULL;
+
+
+    /* check for symmetry type */
+    if (mm_is_general(matcode))
+        types[3] = MM_GENERAL_STR;
+    else
+    if (mm_is_symmetric(matcode))
+        types[3] = MM_SYMM_STR;
+    else 
+    if (mm_is_hermitian(matcode))
+        types[3] = MM_HERM_STR;
+    else 
+    if (mm_is_skew(matcode))
+        types[3] = MM_SKEW_STR;
+    else
+        return NULL;
+
+    sprintf(buffer,"%s %s %s %s", types[0], types[1], types[2], types[3]);
+    return mm_strdup(buffer);
+
+}

Samples/4_CUDA_Libraries/cuSolverRf/mmio.h

@@ -0,0 +1,141 @@
+/* 
+*   Matrix Market I/O library for ANSI C
+*
+*   See http://math.nist.gov/MatrixMarket for details.
+*
+*
+*/
+
+#ifndef MM_IO_H
+#define MM_IO_H
+
+#if defined(__cplusplus)
+extern "C" {
+#endif /* __cplusplus */
+
+#define MM_MAX_LINE_LENGTH 1025
+#define MatrixMarketBanner "%%MatrixMarket"
+#define MM_MAX_TOKEN_LENGTH 64
+
+typedef char MM_typecode[4];
+
+char *mm_typecode_to_str(MM_typecode matcode);
+
+int mm_read_banner(FILE *f, MM_typecode *matcode);
+int mm_read_mtx_crd_size(FILE *f, int *M, int *N, int *nz);
+int mm_read_mtx_array_size(FILE *f, int *M, int *N);
+
+int mm_write_banner(FILE *f, MM_typecode matcode);
+int mm_write_mtx_crd_size(FILE *f, int M, int N, int nz);
+int mm_write_mtx_array_size(FILE *f, int M, int N);
+
+
+/********************* MM_typecode query fucntions ***************************/
+
+#define mm_is_matrix(typecode)	((typecode)[0]=='M')
+
+#define mm_is_sparse(typecode)	((typecode)[1]=='C')
+#define mm_is_coordinate(typecode)((typecode)[1]=='C')
+#define mm_is_dense(typecode)	((typecode)[1]=='A')
+#define mm_is_array(typecode)	((typecode)[1]=='A')
+
+#define mm_is_complex(typecode)	((typecode)[2]=='C')
+#define mm_is_real(typecode)		((typecode)[2]=='R')
+#define mm_is_pattern(typecode)	((typecode)[2]=='P')
+#define mm_is_integer(typecode) ((typecode)[2]=='I')
+
+#define mm_is_symmetric(typecode)((typecode)[3]=='S')
+#define mm_is_general(typecode)	((typecode)[3]=='G')
+#define mm_is_skew(typecode)	((typecode)[3]=='K')
+#define mm_is_hermitian(typecode)((typecode)[3]=='H')
+
+int mm_is_valid(MM_typecode matcode);		/* too complex for a macro */
+
+
+/********************* MM_typecode modify fucntions ***************************/
+
+#define mm_set_matrix(typecode)	((*typecode)[0]='M')
+#define mm_set_coordinate(typecode)	((*typecode)[1]='C')
+#define mm_set_array(typecode)	((*typecode)[1]='A')
+#define mm_set_dense(typecode)	mm_set_array(typecode)
+#define mm_set_sparse(typecode)	mm_set_coordinate(typecode)
+
+#define mm_set_complex(typecode)((*typecode)[2]='C')
+#define mm_set_real(typecode)	((*typecode)[2]='R')
+#define mm_set_pattern(typecode)((*typecode)[2]='P')
+#define mm_set_integer(typecode)((*typecode)[2]='I')
+
+
+#define mm_set_symmetric(typecode)((*typecode)[3]='S')
+#define mm_set_general(typecode)((*typecode)[3]='G')
+#define mm_set_skew(typecode)	((*typecode)[3]='K')
+#define mm_set_hermitian(typecode)((*typecode)[3]='H')
+
+#define mm_clear_typecode(typecode) ((*typecode)[0]=(*typecode)[1]= \
+									(*typecode)[2]=' ',(*typecode)[3]='G')
+
+#define mm_initialize_typecode(typecode) mm_clear_typecode(typecode)
+
+
+/********************* Matrix Market error codes ***************************/
+
+
+#define MM_COULD_NOT_READ_FILE	11
+#define MM_PREMATURE_EOF		12
+#define MM_NOT_MTX				13
+#define MM_NO_HEADER			14
+#define MM_UNSUPPORTED_TYPE		15
+#define MM_LINE_TOO_LONG		16
+#define MM_COULD_NOT_WRITE_FILE	17
+
+
+/******************** Matrix Market internal definitions ********************
+
+   MM_matrix_typecode: 4-character sequence
+
+				    ojbect 		sparse/   	data        storage 
+						  		dense     	type        scheme
+
+   string position:	 [0]        [1]			[2]         [3]
+
+   Matrix typecode:  M(atrix)  C(oord)		R(eal)   	G(eneral)
+						        A(array)	C(omplex)   H(ermitian)
+											P(attern)   S(ymmetric)
+								    		I(nteger)	K(kew)
+
+ ***********************************************************************/
+
+#define MM_MTX_STR		"matrix"
+#define MM_ARRAY_STR	"array"
+#define MM_DENSE_STR	"array"
+#define MM_COORDINATE_STR "coordinate" 
+#define MM_SPARSE_STR	"coordinate"
+#define MM_COMPLEX_STR	"complex"
+#define MM_REAL_STR		"real"
+#define MM_INT_STR		"integer"
+#define MM_GENERAL_STR  "general"
+#define MM_SYMM_STR		"symmetric"
+#define MM_HERM_STR		"hermitian"
+#define MM_SKEW_STR		"skew-symmetric"
+#define MM_PATTERN_STR  "pattern"
+
+
+/*  high level routines */
+int mm_read_mtx_crd(char *fname, int *M, int *N, int *nz, int **I, int **J, 
+        double **val, MM_typecode *matcode);
+
+int mm_write_mtx_crd(char fname[], int M, int N, int nz, int I[], int J[],
+		 double val[], MM_typecode matcode);
+int mm_read_mtx_crd_data(FILE *f, int M, int N, int nz, int I[], int J[],
+		double val[], MM_typecode matcode);
+int mm_read_mtx_crd_entry(FILE *f, int *I, int *J, double *real, double *img,
+			MM_typecode matcode);
+
+int mm_read_unsymmetric_sparse(const char *fname, int *M_, int *N_, int *nz_,
+                double **val_, int **I_, int **J_);
+
+#if defined(__cplusplus)
+}
+#endif /* __cplusplus */  
+
+#endif

Samples/4_CUDA_Libraries/cuSolverRf/mmio_wrapper.cpp

@@ -0,0 +1,501 @@
+/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <cusolverDn.h>
+#include <math.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "mmio.h"
+
+/* avoid Windows warnings (for example: strcpy, fscanf, etc.) */
+#if defined(_WIN32)
+#define _CRT_SECURE_NO_WARNINGS
+#endif
+
+/* various __inline__ __device__  function to initialize a T_ELEM */
+template <typename T_ELEM>
+__inline__ T_ELEM cuGet(int);
+template <>
+__inline__ float cuGet<float>(int x) {
+  return float(x);
+}
+
+template <>
+__inline__ double cuGet<double>(int x) {
+  return double(x);
+}
+
+template <>
+__inline__ cuComplex cuGet<cuComplex>(int x) {
+  return (make_cuComplex(float(x), 0.0f));
+}
+
+template <>
+__inline__ cuDoubleComplex cuGet<cuDoubleComplex>(int x) {
+  return (make_cuDoubleComplex(double(x), 0.0));
+}
+
+template <typename T_ELEM>
+__inline__ T_ELEM cuGet(int, int);
+template <>
+__inline__ float cuGet<float>(int x, int y) {
+  return float(x);
+}
+
+template <>
+__inline__ double cuGet<double>(int x, int y) {
+  return double(x);
+}
+
+template <>
+__inline__ cuComplex cuGet<cuComplex>(int x, int y) {
+  return make_cuComplex(float(x), float(y));
+}
+
+template <>
+__inline__ cuDoubleComplex cuGet<cuDoubleComplex>(int x, int y) {
+  return (make_cuDoubleComplex(double(x), double(y)));
+}
+
+template <typename T_ELEM>
+__inline__ T_ELEM cuGet(float);
+template <>
+__inline__ float cuGet<float>(float x) {
+  return float(x);
+}
+
+template <>
+__inline__ double cuGet<double>(float x) {
+  return double(x);
+}
+
+template <>
+__inline__ cuComplex cuGet<cuComplex>(float x) {
+  return (make_cuComplex(float(x), 0.0f));
+}
+
+template <>
+__inline__ cuDoubleComplex cuGet<cuDoubleComplex>(float x) {
+  return (make_cuDoubleComplex(double(x), 0.0));
+}
+
+template <typename T_ELEM>
+__inline__ T_ELEM cuGet(float, float);
+template <>
+__inline__ float cuGet<float>(float x, float y) {
+  return float(x);
+}
+
+template <>
+__inline__ double cuGet<double>(float x, float y) {
+  return double(x);
+}
+
+template <>
+__inline__ cuComplex cuGet<cuComplex>(float x, float y) {
+  return (make_cuComplex(float(x), float(y)));
+}
+
+template <>
+__inline__ cuDoubleComplex cuGet<cuDoubleComplex>(float x, float y) {
+  return (make_cuDoubleComplex(double(x), double(y)));
+}
+
+template <typename T_ELEM>
+__inline__ T_ELEM cuGet(double);
+template <>
+__inline__ float cuGet<float>(double x) {
+  return float(x);
+}
+
+template <>
+__inline__ double cuGet<double>(double x) {
+  return double(x);
+}
+
+template <>
+__inline__ cuComplex cuGet<cuComplex>(double x) {
+  return (make_cuComplex(float(x), 0.0f));
+}
+
+template <>
+__inline__ cuDoubleComplex cuGet<cuDoubleComplex>(double x) {
+  return (make_cuDoubleComplex(double(x), 0.0));
+}
+
+template <typename T_ELEM>
+__inline__ T_ELEM cuGet(double, double);
+template <>
+__inline__ float cuGet<float>(double x, double y) {
+  return float(x);
+}
+
+template <>
+__inline__ double cuGet<double>(double x, double y) {
+  return double(x);
+}
+
+template <>
+__inline__ cuComplex cuGet<cuComplex>(double x, double y) {
+  return (make_cuComplex(float(x), float(y)));
+}
+
+template <>
+__inline__ cuDoubleComplex cuGet<cuDoubleComplex>(double x, double y) {
+  return (make_cuDoubleComplex(double(x), double(y)));
+}
+
+static void compress_index(const int *Ind, int nnz, int m, int *Ptr, int base) {
+  int i;
+
+  /* initialize everything to zero */
+  for (i = 0; i < m + 1; i++) {
+    Ptr[i] = 0;
+  }
+  /* count elements in every row */
+  Ptr[0] = base;
+  for (i = 0; i < nnz; i++) {
+    Ptr[Ind[i] + (1 - base)]++;
+  }
+  /* add all the values */
+  for (i = 0; i < m; i++) {
+    Ptr[i + 1] += Ptr[i];
+  }
+}
+
+struct cooFormat {
+  int i;
+  int j;
+  int p;  // permutation
+};
+
+int cmp_cooFormat_csr(struct cooFormat *s, struct cooFormat *t) {
+  if (s->i < t->i) {
+    return -1;
+  } else if (s->i > t->i) {
+    return 1;
+  } else {
+    return s->j - t->j;
+  }
+}
+
+int cmp_cooFormat_csc(struct cooFormat *s, struct cooFormat *t) {
+  if (s->j < t->j) {
+    return -1;
+  } else if (s->j > t->j) {
+    return 1;
+  } else {
+    return s->i - t->i;
+  }
+}
+
+typedef int (*FUNPTR)(const void *, const void *);
+typedef int (*FUNPTR2)(struct cooFormat *s, struct cooFormat *t);
+
+static FUNPTR2 fptr_array[2] = {
+    cmp_cooFormat_csr,
+    cmp_cooFormat_csc,
+};
+
+static int verify_pattern(int m, int nnz, int *csrRowPtr, int *csrColInd) {
+  int i, col, start, end, base_index;
+  int error_found = 0;
+
+  if (nnz != (csrRowPtr[m] - csrRowPtr[0])) {
+    fprintf(stderr,
+            "Error (nnz check failed): (csrRowPtr[%d]=%d - csrRowPtr[%d]=%d) "
+            "!= (nnz=%d)\n",
+            0, csrRowPtr[0], m, csrRowPtr[m], nnz);
+    error_found = 1;
+  }
+
+  base_index = csrRowPtr[0];
+  if ((0 != base_index) && (1 != base_index)) {
+    fprintf(stderr, "Error (base index check failed): base index = %d\n",
+            base_index);
+    error_found = 1;
+  }
+
+  for (i = 0; (!error_found) && (i < m); i++) {
+    start = csrRowPtr[i] - base_index;
+    end = csrRowPtr[i + 1] - base_index;
+    if (start > end) {
+      fprintf(
+          stderr,
+          "Error (corrupted row): csrRowPtr[%d] (=%d) > csrRowPtr[%d] (=%d)\n",
+          i, start + base_index, i + 1, end + base_index);
+      error_found = 1;
+    }
+    for (col = start; col < end; col++) {
+      if (csrColInd[col] < base_index) {
+        fprintf(
+            stderr,
+            "Error (column vs. base index check failed): csrColInd[%d] < %d\n",
+            col, base_index);
+        error_found = 1;
+      }
+      if ((col < (end - 1)) && (csrColInd[col] >= csrColInd[col + 1])) {
+        fprintf(stderr,
+                "Error (sorting of the column indecis check failed): "
+                "(csrColInd[%d]=%d) >= (csrColInd[%d]=%d)\n",
+                col, csrColInd[col], col + 1, csrColInd[col + 1]);
+        error_found = 1;
+      }
+    }
+  }
+  return error_found;
+}
+
+template <typename T_ELEM>
+int loadMMSparseMatrix(char *filename, char elem_type, bool csrFormat, int *m,
+                       int *n, int *nnz, T_ELEM **aVal, int **aRowInd,
+                       int **aColInd, int extendSymMatrix) {
+  MM_typecode matcode;
+  double *tempVal;
+  int *tempRowInd, *tempColInd;
+  double *tval;
+  int *trow, *tcol;
+  int *csrRowPtr, *cscColPtr;
+  int i, j, error, base, count;
+  struct cooFormat *work;
+
+  /* read the matrix */
+  error = mm_read_mtx_crd(filename, m, n, nnz, &trow, &tcol, &tval, &matcode);
+  if (error) {
+    fprintf(stderr, "!!!! can not open file: '%s'\n", filename);
+    return 1;
+  }
+
+  /* start error checking */
+  if (mm_is_complex(matcode) && ((elem_type != 'z') && (elem_type != 'c'))) {
+    fprintf(stderr, "!!!! complex matrix requires type 'z' or 'c'\n");
+    return 1;
+  }
+
+  if (mm_is_dense(matcode) || mm_is_array(matcode) ||
+      mm_is_pattern(matcode) /*|| mm_is_integer(matcode)*/) {
+    fprintf(
+        stderr,
+        "!!!! dense, array, pattern and integer matrices are not supported\n");
+    return 1;
+  }
+
+  /* if necessary symmetrize the pattern (transform from triangular to full) */
+  if ((extendSymMatrix) && (mm_is_symmetric(matcode) ||
+                            mm_is_hermitian(matcode) || mm_is_skew(matcode))) {
+    // count number of non-diagonal elements
+    count = 0;
+    for (i = 0; i < (*nnz); i++) {
+      if (trow[i] != tcol[i]) {
+        count++;
+      }
+    }
+    // allocate space for the symmetrized matrix
+    tempRowInd = (int *)malloc((*nnz + count) * sizeof(int));
+    tempColInd = (int *)malloc((*nnz + count) * sizeof(int));
+    if (mm_is_real(matcode) || mm_is_integer(matcode)) {
+      tempVal = (double *)malloc((*nnz + count) * sizeof(double));
+    } else {
+      tempVal = (double *)malloc(2 * (*nnz + count) * sizeof(double));
+    }
+    // copy the elements regular and transposed locations
+    for (j = 0, i = 0; i < (*nnz); i++) {
+      tempRowInd[j] = trow[i];
+      tempColInd[j] = tcol[i];
+      if (mm_is_real(matcode) || mm_is_integer(matcode)) {
+        tempVal[j] = tval[i];
+      } else {
+        tempVal[2 * j] = tval[2 * i];
+        tempVal[2 * j + 1] = tval[2 * i + 1];
+      }
+      j++;
+      if (trow[i] != tcol[i]) {
+        tempRowInd[j] = tcol[i];
+        tempColInd[j] = trow[i];
+        if (mm_is_real(matcode) || mm_is_integer(matcode)) {
+          if (mm_is_skew(matcode)) {
+            tempVal[j] = -tval[i];
+          } else {
+            tempVal[j] = tval[i];
+          }
+        } else {
+          if (mm_is_hermitian(matcode)) {
+            tempVal[2 * j] = tval[2 * i];
+            tempVal[2 * j + 1] = -tval[2 * i + 1];
+          } else {
+            tempVal[2 * j] = tval[2 * i];
+            tempVal[2 * j + 1] = tval[2 * i + 1];
+          }
+        }
+        j++;
+      }
+    }
+    (*nnz) += count;
+    // free temporary storage
+    free(trow);
+    free(tcol);
+    free(tval);
+  } else {
+    tempRowInd = trow;
+    tempColInd = tcol;
+    tempVal = tval;
+  }
+  // life time of (trow, tcol, tval) is over.
+  // please use COO format (tempRowInd, tempColInd, tempVal)
+
+  // use qsort to sort COO format
+  work = (struct cooFormat *)malloc(sizeof(struct cooFormat) * (*nnz));
+  if (NULL == work) {
+    fprintf(stderr, "!!!! allocation error, malloc failed\n");
+    return 1;
+  }
+  for (i = 0; i < (*nnz); i++) {
+    work[i].i = tempRowInd[i];
+    work[i].j = tempColInd[i];
+    work[i].p = i;  // permutation is identity
+  }
+
+  if (csrFormat) {
+    /* create row-major ordering of indices (sorted by row and within each row
+     * by column) */
+    qsort(work, *nnz, sizeof(struct cooFormat), (FUNPTR)fptr_array[0]);
+  } else {
+    /* create column-major ordering of indices (sorted by column and within each
+     * column by row) */
+    qsort(work, *nnz, sizeof(struct cooFormat), (FUNPTR)fptr_array[1]);
+  }
+
+  // (tempRowInd, tempColInd) is sorted either by row-major or by col-major
+  for (i = 0; i < (*nnz); i++) {
+    tempRowInd[i] = work[i].i;
+    tempColInd[i] = work[i].j;
+  }
+
+  // setup base
+  // check if there is any row/col 0, if so base-0
+  // check if there is any row/col equal to matrix dimension m/n, if so base-1
+  int base0 = 0;
+  int base1 = 0;
+  for (i = 0; i < (*nnz); i++) {
+    const int row = tempRowInd[i];
+    const int col = tempColInd[i];
+    if ((0 == row) || (0 == col)) {
+      base0 = 1;
+    }
+    if ((*m == row) || (*n == col)) {
+      base1 = 1;
+    }
+  }
+  if (base0 && base1) {
+    printf("Error: input matrix is base-0 and base-1 \n");
+    return 1;
+  }
+
+  base = 0;
+  if (base1) {
+    base = 1;
+  }
+
+  /* compress the appropriate indices */
+  if (csrFormat) {
+    /* CSR format (assuming row-major format) */
+    csrRowPtr = (int *)malloc(((*m) + 1) * sizeof(csrRowPtr[0]));
+    if (!csrRowPtr) return 1;
+    compress_index(tempRowInd, *nnz, *m, csrRowPtr, base);
+
+    *aRowInd = csrRowPtr;
+    *aColInd = (int *)malloc((*nnz) * sizeof(int));
+  } else {
+    /* CSC format (assuming column-major format) */
+    cscColPtr = (int *)malloc(((*n) + 1) * sizeof(cscColPtr[0]));
+    if (!cscColPtr) return 1;
+    compress_index(tempColInd, *nnz, *n, cscColPtr, base);
+
+    *aColInd = cscColPtr;
+    *aRowInd = (int *)malloc((*nnz) * sizeof(int));
+  }
+
+  /* transfrom the matrix values of type double into one of the cusparse library
+   * types */
+  *aVal = (T_ELEM *)malloc((*nnz) * sizeof(T_ELEM));
+
+  for (i = 0; i < (*nnz); i++) {
+    if (csrFormat) {
+      (*aColInd)[i] = tempColInd[i];
+    } else {
+      (*aRowInd)[i] = tempRowInd[i];
+    }
+    if (mm_is_real(matcode) || mm_is_integer(matcode)) {
+      (*aVal)[i] = cuGet<T_ELEM>(tempVal[work[i].p]);
+    } else {
+      (*aVal)[i] =
+          cuGet<T_ELEM>(tempVal[2 * work[i].p], tempVal[2 * work[i].p + 1]);
+    }
+  }
+
+  /* check for corruption */
+  int error_found;
+  if (csrFormat) {
+    error_found = verify_pattern(*m, *nnz, *aRowInd, *aColInd);
+  } else {
+    error_found = verify_pattern(*n, *nnz, *aColInd, *aRowInd);
+  }
+  if (error_found) {
+    fprintf(stderr, "!!!! verify_pattern failed\n");
+    return 1;
+  }
+
+  /* cleanup and exit */
+  free(work);
+  free(tempVal);
+  free(tempColInd);
+  free(tempRowInd);
+
+  return 0;
+}
+
+/* specific instantiation */
+template int loadMMSparseMatrix<float>(char *filename, char elem_type,
+                                       bool csrFormat, int *m, int *n, int *nnz,
+                                       float **aVal, int **aRowInd,
+                                       int **aColInd, int extendSymMatrix);
+
+template int loadMMSparseMatrix<double>(char *filename, char elem_type,
+                                        bool csrFormat, int *m, int *n,
+                                        int *nnz, double **aVal, int **aRowInd,
+                                        int **aColInd, int extendSymMatrix);
+
+template int loadMMSparseMatrix<cuComplex>(char *filename, char elem_type,
+                                           bool csrFormat, int *m, int *n,
+                                           int *nnz, cuComplex **aVal,
+                                           int **aRowInd, int **aColInd,
+                                           int extendSymMatrix);
+
+template int loadMMSparseMatrix<cuDoubleComplex>(
+    char *filename, char elem_type, bool csrFormat, int *m, int *n, int *nnz,
+    cuDoubleComplex **aVal, int **aRowInd, int **aColInd, int extendSymMatrix);