add and update samples for CUDA 11.6

Samples/4_CUDA_Libraries/cuSolverSp_LowlevelCholesky/.vscode/c_cpp_properties.json

@@ -0,0 +1,18 @@
+{
+    "configurations": [
+        {
+            "name": "Linux",
+            "includePath": [
+                "${workspaceFolder}/**",
+                "${workspaceFolder}/../../../Common"
+            ],
+            "defines": [],
+            "compilerPath": "/usr/local/cuda/bin/nvcc",
+            "cStandard": "gnu17",
+            "cppStandard": "gnu++14",
+            "intelliSenseMode": "linux-gcc-x64",
+            "configurationProvider": "ms-vscode.makefile-tools"
+        }
+    ],
+    "version": 4
+}

Samples/4_CUDA_Libraries/cuSolverSp_LowlevelCholesky/.vscode/extensions.json

@@ -0,0 +1,7 @@
+{
+    "recommendations": [
+        "nvidia.nsight-vscode-edition",
+        "ms-vscode.cpptools",
+        "ms-vscode.makefile-tools"
+    ]
+}

Samples/4_CUDA_Libraries/cuSolverSp_LowlevelCholesky/.vscode/launch.json

@@ -0,0 +1,10 @@
+{
+    "configurations": [
+        {
+            "name": "CUDA C++: Launch",
+            "type": "cuda-gdb",
+            "request": "launch",
+            "program": "${workspaceFolder}/cuSolverSp_LowlevelCholesky"
+        }
+    ]
+}

Samples/4_CUDA_Libraries/cuSolverSp_LowlevelCholesky/.vscode/tasks.json

@@ -0,0 +1,15 @@
+{
+    "version": "2.0.0",
+    "tasks": [
+        {
+            "label": "sample",
+            "type": "shell",
+            "command": "make dbg=1",
+            "problemMatcher": ["$nvcc"],
+            "group": {
+                "kind": "build",
+                "isDefault": true
+            }
+        }
+    ]
+}

Samples/4_CUDA_Libraries/cuSolverSp_LowlevelCholesky/Makefile

@@ -0,0 +1,352 @@
+################################################################################
+# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#  * Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#  * Neither the name of NVIDIA CORPORATION nor the names of its
+#    contributors may be used to endorse or promote products derived
+#    from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+################################################################################
+#
+# Makefile project only supported on Mac OS X and Linux Platforms)
+#
+################################################################################
+
+# Location of the CUDA Toolkit
+CUDA_PATH ?= /usr/local/cuda
+
+##############################
+# start deprecated interface #
+##############################
+ifeq ($(x86_64),1)
+    $(info WARNING - x86_64 variable has been deprecated)
+    $(info WARNING - please use TARGET_ARCH=x86_64 instead)
+    TARGET_ARCH ?= x86_64
+endif
+ifeq ($(ARMv7),1)
+    $(info WARNING - ARMv7 variable has been deprecated)
+    $(info WARNING - please use TARGET_ARCH=armv7l instead)
+    TARGET_ARCH ?= armv7l
+endif
+ifeq ($(aarch64),1)
+    $(info WARNING - aarch64 variable has been deprecated)
+    $(info WARNING - please use TARGET_ARCH=aarch64 instead)
+    TARGET_ARCH ?= aarch64
+endif
+ifeq ($(ppc64le),1)
+    $(info WARNING - ppc64le variable has been deprecated)
+    $(info WARNING - please use TARGET_ARCH=ppc64le instead)
+    TARGET_ARCH ?= ppc64le
+endif
+ifneq ($(GCC),)
+    $(info WARNING - GCC variable has been deprecated)
+    $(info WARNING - please use HOST_COMPILER=$(GCC) instead)
+    HOST_COMPILER ?= $(GCC)
+endif
+ifneq ($(abi),)
+    $(error ERROR - abi variable has been removed)
+endif
+############################
+# end deprecated interface #
+############################
+
+# architecture
+HOST_ARCH   := $(shell uname -m)
+TARGET_ARCH ?= $(HOST_ARCH)
+ifneq (,$(filter $(TARGET_ARCH),x86_64 aarch64 sbsa ppc64le armv7l))
+    ifneq ($(TARGET_ARCH),$(HOST_ARCH))
+        ifneq (,$(filter $(TARGET_ARCH),x86_64 aarch64 sbsa ppc64le))
+            TARGET_SIZE := 64
+        else ifneq (,$(filter $(TARGET_ARCH),armv7l))
+            TARGET_SIZE := 32
+        endif
+    else
+        TARGET_SIZE := $(shell getconf LONG_BIT)
+    endif
+else
+    $(error ERROR - unsupported value $(TARGET_ARCH) for TARGET_ARCH!)
+endif
+
+# sbsa and aarch64 systems look similar. Need to differentiate them at host level for now.
+ifeq ($(HOST_ARCH),aarch64)
+    ifeq ($(CUDA_PATH)/targets/sbsa-linux,$(shell ls -1d $(CUDA_PATH)/targets/sbsa-linux 2>/dev/null))
+        HOST_ARCH := sbsa
+        TARGET_ARCH := sbsa
+    endif
+endif
+
+ifneq ($(TARGET_ARCH),$(HOST_ARCH))
+    ifeq (,$(filter $(HOST_ARCH)-$(TARGET_ARCH),aarch64-armv7l x86_64-armv7l x86_64-aarch64 x86_64-sbsa x86_64-ppc64le))
+        $(error ERROR - cross compiling from $(HOST_ARCH) to $(TARGET_ARCH) is not supported!)
+    endif
+endif
+
+# When on native aarch64 system with userspace of 32-bit, change TARGET_ARCH to armv7l
+ifeq ($(HOST_ARCH)-$(TARGET_ARCH)-$(TARGET_SIZE),aarch64-aarch64-32)
+    TARGET_ARCH = armv7l
+endif
+
+# operating system
+HOST_OS   := $(shell uname -s 2>/dev/null | tr "[:upper:]" "[:lower:]")
+TARGET_OS ?= $(HOST_OS)
+ifeq (,$(filter $(TARGET_OS),linux darwin qnx android))
+    $(error ERROR - unsupported value $(TARGET_OS) for TARGET_OS!)
+endif
+
+# host compiler
+ifeq ($(TARGET_OS),darwin)
+    ifeq ($(shell expr `xcodebuild -version | grep -i xcode | awk '{print $$2}' | cut -d'.' -f1` \>= 5),1)
+        HOST_COMPILER ?= clang++
+    endif
+else ifneq ($(TARGET_ARCH),$(HOST_ARCH))
+    ifeq ($(HOST_ARCH)-$(TARGET_ARCH),x86_64-armv7l)
+        ifeq ($(TARGET_OS),linux)
+            HOST_COMPILER ?= arm-linux-gnueabihf-g++
+        else ifeq ($(TARGET_OS),qnx)
+            ifeq ($(QNX_HOST),)
+                $(error ERROR - QNX_HOST must be passed to the QNX host toolchain)
+            endif
+            ifeq ($(QNX_TARGET),)
+                $(error ERROR - QNX_TARGET must be passed to the QNX target toolchain)
+            endif
+            export QNX_HOST
+            export QNX_TARGET
+            HOST_COMPILER ?= $(QNX_HOST)/usr/bin/arm-unknown-nto-qnx6.6.0eabi-g++
+        else ifeq ($(TARGET_OS),android)
+            HOST_COMPILER ?= arm-linux-androideabi-g++
+        endif
+    else ifeq ($(TARGET_ARCH),aarch64)
+        ifeq ($(TARGET_OS), linux)
+            HOST_COMPILER ?= aarch64-linux-gnu-g++
+        else ifeq ($(TARGET_OS),qnx)
+            ifeq ($(QNX_HOST),)
+                $(error ERROR - QNX_HOST must be passed to the QNX host toolchain)
+            endif
+            ifeq ($(QNX_TARGET),)
+                $(error ERROR - QNX_TARGET must be passed to the QNX target toolchain)
+            endif
+            export QNX_HOST
+            export QNX_TARGET
+            HOST_COMPILER ?= $(QNX_HOST)/usr/bin/q++
+        else ifeq ($(TARGET_OS), android)
+            HOST_COMPILER ?= aarch64-linux-android-clang++
+        endif
+    else ifeq ($(TARGET_ARCH),sbsa)
+        HOST_COMPILER ?= aarch64-linux-gnu-g++
+    else ifeq ($(TARGET_ARCH),ppc64le)
+        HOST_COMPILER ?= powerpc64le-linux-gnu-g++
+    endif
+endif
+HOST_COMPILER ?= g++
+NVCC          := $(CUDA_PATH)/bin/nvcc -ccbin $(HOST_COMPILER)
+
+# internal flags
+NVCCFLAGS   := -m${TARGET_SIZE}
+CCFLAGS     :=
+LDFLAGS     :=
+
+# build flags
+ifeq ($(TARGET_OS),darwin)
+    LDFLAGS += -rpath $(CUDA_PATH)/lib
+    CCFLAGS += -arch $(HOST_ARCH)
+else ifeq ($(HOST_ARCH)-$(TARGET_ARCH)-$(TARGET_OS),x86_64-armv7l-linux)
+    LDFLAGS += --dynamic-linker=/lib/ld-linux-armhf.so.3
+    CCFLAGS += -mfloat-abi=hard
+else ifeq ($(TARGET_OS),android)
+    LDFLAGS += -pie
+    CCFLAGS += -fpie -fpic -fexceptions
+endif
+
+ifneq ($(TARGET_ARCH),$(HOST_ARCH))
+    ifeq ($(TARGET_ARCH)-$(TARGET_OS),armv7l-linux)
+        ifneq ($(TARGET_FS),)
+            GCCVERSIONLTEQ46 := $(shell expr `$(HOST_COMPILER) -dumpversion` \<= 4.6)
+            ifeq ($(GCCVERSIONLTEQ46),1)
+                CCFLAGS += --sysroot=$(TARGET_FS)
+            endif
+            LDFLAGS += --sysroot=$(TARGET_FS)
+            LDFLAGS += -rpath-link=$(TARGET_FS)/lib
+            LDFLAGS += -rpath-link=$(TARGET_FS)/usr/lib
+            LDFLAGS += -rpath-link=$(TARGET_FS)/usr/lib/arm-linux-gnueabihf
+        endif
+    endif
+    ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-linux)
+        ifneq ($(TARGET_FS),)
+            GCCVERSIONLTEQ46 := $(shell expr `$(HOST_COMPILER) -dumpversion` \<= 4.6)
+            ifeq ($(GCCVERSIONLTEQ46),1)
+                CCFLAGS += --sysroot=$(TARGET_FS)
+            endif
+            LDFLAGS += --sysroot=$(TARGET_FS)
+            LDFLAGS += -rpath-link=$(TARGET_FS)/lib -L$(TARGET_FS)/lib
+            LDFLAGS += -rpath-link=$(TARGET_FS)/lib/aarch64-linux-gnu -L$(TARGET_FS)/lib/aarch64-linux-gnu
+            LDFLAGS += -rpath-link=$(TARGET_FS)/usr/lib -L$(TARGET_FS)/usr/lib
+            LDFLAGS += -rpath-link=$(TARGET_FS)/usr/lib/aarch64-linux-gnu -L$(TARGET_FS)/usr/lib/aarch64-linux-gnu
+            LDFLAGS += --unresolved-symbols=ignore-in-shared-libs
+            CCFLAGS += -isystem=$(TARGET_FS)/usr/include -I$(TARGET_FS)/usr/include -I$(TARGET_FS)/usr/include/libdrm
+            CCFLAGS += -isystem=$(TARGET_FS)/usr/include/aarch64-linux-gnu -I$(TARGET_FS)/usr/include/aarch64-linux-gnu
+        endif
+    endif
+    ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-qnx)
+        NVCCFLAGS += -D_QNX_SOURCE
+        NVCCFLAGS += --qpp-config 8.3.0,gcc_ntoaarch64le
+        CCFLAGS += -DWIN_INTERFACE_CUSTOM -I/usr/include/aarch64-qnx-gnu
+        LDFLAGS += -lsocket
+        LDFLAGS += -L/usr/lib/aarch64-qnx-gnu
+        CCFLAGS += "-Wl\,-rpath-link\,/usr/lib/aarch64-qnx-gnu"
+        ifdef TARGET_OVERRIDE
+            LDFLAGS += -lslog2
+        endif
+
+        ifneq ($(TARGET_FS),)
+            LDFLAGS += -L$(TARGET_FS)/usr/lib
+            CCFLAGS += "-Wl\,-rpath-link\,$(TARGET_FS)/usr/lib"
+            LDFLAGS += -L$(TARGET_FS)/usr/libnvidia
+            CCFLAGS += "-Wl\,-rpath-link\,$(TARGET_FS)/usr/libnvidia"
+            CCFLAGS += -I$(TARGET_FS)/../include
+        endif
+    endif
+endif
+
+ifdef TARGET_OVERRIDE # cuda toolkit targets override
+    NVCCFLAGS += -target-dir $(TARGET_OVERRIDE)
+endif
+
+# Install directory of different arch
+CUDA_INSTALL_TARGET_DIR :=
+ifeq ($(TARGET_ARCH)-$(TARGET_OS),armv7l-linux)
+    CUDA_INSTALL_TARGET_DIR = targets/armv7-linux-gnueabihf/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-linux)
+    CUDA_INSTALL_TARGET_DIR = targets/aarch64-linux/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),sbsa-linux)
+    CUDA_INSTALL_TARGET_DIR = targets/sbsa-linux/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),armv7l-android)
+    CUDA_INSTALL_TARGET_DIR = targets/armv7-linux-androideabi/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-android)
+    CUDA_INSTALL_TARGET_DIR = targets/aarch64-linux-androideabi/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),armv7l-qnx)
+    CUDA_INSTALL_TARGET_DIR = targets/ARMv7-linux-QNX/
+else ifeq ($(TARGET_ARCH)-$(TARGET_OS),aarch64-qnx)
+    CUDA_INSTALL_TARGET_DIR = targets/aarch64-qnx/
+else ifeq ($(TARGET_ARCH),ppc64le)
+    CUDA_INSTALL_TARGET_DIR = targets/ppc64le-linux/
+endif
+
+# Debug build flags
+ifeq ($(dbg),1)
+      NVCCFLAGS += -g -G
+      BUILD_TYPE := debug
+else
+      BUILD_TYPE := release
+endif
+
+ALL_CCFLAGS :=
+ALL_CCFLAGS += $(NVCCFLAGS)
+ALL_CCFLAGS += $(EXTRA_NVCCFLAGS)
+ALL_CCFLAGS += $(addprefix -Xcompiler ,$(CCFLAGS))
+ALL_CCFLAGS += $(addprefix -Xcompiler ,$(EXTRA_CCFLAGS))
+
+SAMPLE_ENABLED := 1
+
+ifeq ($(TARGET_OS),linux)
+ALL_CCFLAGS += -Xcompiler \"-Wl,--no-as-needed\"
+endif
+
+ALL_LDFLAGS :=
+ALL_LDFLAGS += $(ALL_CCFLAGS)
+ALL_LDFLAGS += $(addprefix -Xlinker ,$(LDFLAGS))
+ALL_LDFLAGS += $(addprefix -Xlinker ,$(EXTRA_LDFLAGS))
+
+# Common includes and paths for CUDA
+INCLUDES  := -I../../../Common
+LIBRARIES :=
+
+################################################################################
+
+# Gencode arguments
+ifeq ($(TARGET_ARCH),$(filter $(TARGET_ARCH),armv7l aarch64 sbsa))
+SMS ?= 53 61 70 72 75 80 86 87
+else
+SMS ?= 35 37 50 52 60 61 70 75 80 86
+endif
+
+ifeq ($(SMS),)
+$(info >>> WARNING - no SM architectures have been specified - waiving sample <<<)
+SAMPLE_ENABLED := 0
+endif
+
+ifeq ($(GENCODE_FLAGS),)
+# Generate SASS code for each SM architecture listed in $(SMS)
+$(foreach sm,$(SMS),$(eval GENCODE_FLAGS += -gencode arch=compute_$(sm),code=sm_$(sm)))
+
+# Generate PTX code from the highest SM architecture in $(SMS) to guarantee forward-compatibility
+HIGHEST_SM := $(lastword $(sort $(SMS)))
+ifneq ($(HIGHEST_SM),)
+GENCODE_FLAGS += -gencode arch=compute_$(HIGHEST_SM),code=compute_$(HIGHEST_SM)
+endif
+endif
+
+ALL_CCFLAGS += --threads 0 --std=c++11
+
+LIBRARIES += -lcusolver -lcusparse
+
+ifeq ($(SAMPLE_ENABLED),0)
+EXEC ?= @echo "[@]"
+endif
+
+################################################################################
+
+# Target rules
+all: build
+
+build: cuSolverSp_LowlevelCholesky
+
+check.deps:
+ifeq ($(SAMPLE_ENABLED),0)
+	@echo "Sample will be waived due to the above missing dependencies"
+else
+	@echo "Sample is ready - all dependencies have been met"
+endif
+
+cuSolverSp_LowlevelCholesky.o:cuSolverSp_LowlevelCholesky.cpp
+	$(EXEC) $(NVCC) $(INCLUDES) $(ALL_CCFLAGS) $(GENCODE_FLAGS) -o $@ -c $<
+
+mmio.c.o:mmio.c
+	$(EXEC) $(NVCC) $(INCLUDES) $(ALL_CCFLAGS) $(GENCODE_FLAGS) -o $@ -c $<
+
+mmio_wrapper.o:mmio_wrapper.cpp
+	$(EXEC) $(NVCC) $(INCLUDES) $(ALL_CCFLAGS) $(GENCODE_FLAGS) -o $@ -c $<
+
+cuSolverSp_LowlevelCholesky: cuSolverSp_LowlevelCholesky.o mmio.c.o mmio_wrapper.o
+	$(EXEC) $(NVCC) $(ALL_LDFLAGS) $(GENCODE_FLAGS) -o $@ $+ $(LIBRARIES)
+	$(EXEC) mkdir -p ../../../bin/$(TARGET_ARCH)/$(TARGET_OS)/$(BUILD_TYPE)
+	$(EXEC) cp $@ ../../../bin/$(TARGET_ARCH)/$(TARGET_OS)/$(BUILD_TYPE)
+
+run: build
+	$(EXEC) ./cuSolverSp_LowlevelCholesky
+
+testrun: build
+
+clean:
+	rm -f cuSolverSp_LowlevelCholesky cuSolverSp_LowlevelCholesky.o mmio.c.o mmio_wrapper.o
+	rm -rf ../../../bin/$(TARGET_ARCH)/$(TARGET_OS)/$(BUILD_TYPE)/cuSolverSp_LowlevelCholesky
+
+clobber: clean

Samples/4_CUDA_Libraries/cuSolverSp_LowlevelCholesky/NsightEclipse.xml

@@ -0,0 +1,90 @@
+<?xml version="1.0" encoding="UTF-8"?> 
+<!DOCTYPE entry SYSTEM "SamplesInfo.dtd">
+<entry>
+  <name>cuSolverSp_LowlevelCholesky</name>
+  <cuda_api_list>
+    <driver>cuDoubleComplex</driver>
+    <driver>cuComplex</driver>
+    <toolkit>cudaStreamDestroy</toolkit>
+    <toolkit>cudaFree</toolkit>
+    <toolkit>cudaStreamCreate</toolkit>
+    <toolkit>cudaMalloc</toolkit>
+    <toolkit>cudaMemcpy</toolkit>
+  </cuda_api_list>
+  <description><![CDATA[A CUDA Sample that demonstrates Cholesky factorization using cuSolverSP's low level APIs.]]></description>
+  <devicecompilation>whole</devicecompilation>
+  <files>
+    <file>lap2D_5pt_n100.mtx</file>
+    <file>lap3D_7pt_n20.mtx</file>
+  </files>
+  <includepaths>
+    <path>./</path>
+    <path>../</path>
+    <path>../../../Common</path>
+  </includepaths>
+  <keyconcepts>
+    <concept level="basic">Linear Algebra</concept>
+    <concept level="basic">CUSOLVER Library</concept>
+  </keyconcepts>
+  <keywords>
+    <keyword>CUSOLVER</keyword>
+    <keyword>Linear Algebra</keyword>
+  </keywords>
+  <libraries>
+    <library>cusolver</library>
+    <library>cusparse</library>
+  </libraries>
+  <librarypaths>
+  </librarypaths>
+  <nsight_eclipse>true</nsight_eclipse>
+  <primary_file>cuSolverSp_LowlevelCholesky.cpp</primary_file>
+  <required_dependencies>
+    <dependency>CUSOLVER</dependency>
+    <dependency>CUSPARSE</dependency>
+  </required_dependencies>
+  <scopes>
+    <scope>1:CUDA Basic Topics</scope>
+    <scope>3:Linear Algebra</scope>
+  </scopes>
+  <sm-arch>sm35</sm-arch>
+  <sm-arch>sm37</sm-arch>
+  <sm-arch>sm50</sm-arch>
+  <sm-arch>sm52</sm-arch>
+  <sm-arch>sm53</sm-arch>
+  <sm-arch>sm60</sm-arch>
+  <sm-arch>sm61</sm-arch>
+  <sm-arch>sm70</sm-arch>
+  <sm-arch>sm72</sm-arch>
+  <sm-arch>sm75</sm-arch>
+  <sm-arch>sm80</sm-arch>
+  <sm-arch>sm86</sm-arch>
+  <sm-arch>sm87</sm-arch>
+  <supported_envs>
+    <env>
+      <arch>x86_64</arch>
+      <platform>linux</platform>
+    </env>
+    <env>
+      <platform>windows7</platform>
+    </env>
+    <env>
+      <arch>x86_64</arch>
+      <platform>macosx</platform>
+    </env>
+    <env>
+      <arch>ppc64le</arch>
+      <platform>linux</platform>
+    </env>
+    <env>
+      <arch>arm</arch>
+    </env>
+    <env>
+      <arch>sbsa</arch>
+    </env>
+  </supported_envs>
+  <supported_sm_architectures>
+    <include>all</include>
+  </supported_sm_architectures>
+  <title>cuSolverSp LowlevelCholesky Solver</title>
+  <type>exe</type>
+</entry>

Samples/4_CUDA_Libraries/cuSolverSp_LowlevelCholesky/README.md

@@ -0,0 +1,77 @@
+# cuSolverSp_LowlevelCholesky - cuSolverSp LowlevelCholesky Solver
+
+## Description
+
+A CUDA Sample that demonstrates Cholesky factorization using cuSolverSP's low level APIs.
+
+## Key Concepts
+
+Linear Algebra, CUSOLVER Library
+
+## Supported SM Architectures
+
+[SM 3.5 ](https://developer.nvidia.com/cuda-gpus)  [SM 3.7 ](https://developer.nvidia.com/cuda-gpus)  [SM 5.0 ](https://developer.nvidia.com/cuda-gpus)  [SM 5.2 ](https://developer.nvidia.com/cuda-gpus)  [SM 5.3 ](https://developer.nvidia.com/cuda-gpus)  [SM 6.0 ](https://developer.nvidia.com/cuda-gpus)  [SM 6.1 ](https://developer.nvidia.com/cuda-gpus)  [SM 7.0 ](https://developer.nvidia.com/cuda-gpus)  [SM 7.2 ](https://developer.nvidia.com/cuda-gpus)  [SM 7.5 ](https://developer.nvidia.com/cuda-gpus)  [SM 8.0 ](https://developer.nvidia.com/cuda-gpus)  [SM 8.6 ](https://developer.nvidia.com/cuda-gpus)  [SM 8.7 ](https://developer.nvidia.com/cuda-gpus)
+
+## Supported OSes
+
+Linux, Windows
+
+## Supported CPU Architecture
+
+x86_64, ppc64le, armv7l
+
+## CUDA APIs involved
+
+### [CUDA Driver API](http://docs.nvidia.com/cuda/cuda-driver-api/index.html)
+cuDoubleComplex, cuComplex
+
+### [CUDA Runtime API](http://docs.nvidia.com/cuda/cuda-runtime-api/index.html)
+cudaStreamDestroy, cudaFree, cudaStreamCreate, cudaMalloc, cudaMemcpy
+
+## Dependencies needed to build/run
+[CUSOLVER](../../README.md#cusolver), [CUSPARSE](../../README.md#cusparse)
+
+## Prerequisites
+
+Download and install the [CUDA Toolkit 11.6](https://developer.nvidia.com/cuda-downloads) for your corresponding platform.
+Make sure the dependencies mentioned in [Dependencies]() section above are installed.
+
+## Build and Run
+
+### Windows
+The Windows samples are built using the Visual Studio IDE. Solution files (.sln) are provided for each supported version of Visual Studio, using the format:
+```
+*_vs<version>.sln - for Visual Studio <version>
+```
+Each individual sample has its own set of solution files in its directory:
+
+To build/examine all the samples at once, the complete solution files should be used. To build/examine a single sample, the individual sample solution files should be used.
+> **Note:** Some samples require that the Microsoft DirectX SDK (June 2010 or newer) be installed and that the VC++ directory paths are properly set up (**Tools > Options...**). Check DirectX Dependencies section for details."
+
+### Linux
+The Linux samples are built using makefiles. To use the makefiles, change the current directory to the sample directory you wish to build, and run make:
+```
+$ cd <sample_dir>
+$ make
+```
+The samples makefiles can take advantage of certain options:
+*  **TARGET_ARCH=<arch>** - cross-compile targeting a specific architecture. Allowed architectures are x86_64, ppc64le, armv7l.
+    By default, TARGET_ARCH is set to HOST_ARCH. On a x86_64 machine, not setting TARGET_ARCH is the equivalent of setting TARGET_ARCH=x86_64.<br/>
+`$ make TARGET_ARCH=x86_64` <br/> `$ make TARGET_ARCH=ppc64le` <br/> `$ make TARGET_ARCH=armv7l` <br/>
+    See [here](http://docs.nvidia.com/cuda/cuda-samples/index.html#cross-samples) for more details.
+*   **dbg=1** - build with debug symbols
+    ```
+    $ make dbg=1
+    ```
+*   **SMS="A B ..."** - override the SM architectures for which the sample will be built, where `"A B ..."` is a space-delimited list of SM architectures. For example, to generate SASS for SM 50 and SM 60, use `SMS="50 60"`.
+    ```
+    $ make SMS="50 60"
+    ```
+
+*  **HOST_COMPILER=<host_compiler>** - override the default g++ host compiler. See the [Linux Installation Guide](http://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html#system-requirements) for a list of supported host compilers.
+```
+    $ make HOST_COMPILER=g++
+```
+
+## References (for more details)
+

Samples/4_CUDA_Libraries/cuSolverSp_LowlevelCholesky/cuSolverSp_LowlevelCholesky.cpp

@@ -0,0 +1,400 @@
+/*
+ * Copyright 2015 NVIDIA Corporation.  All rights reserved.
+ *
+ * Please refer to the NVIDIA end user license agreement (EULA) associated
+ * with this source code for terms and conditions that govern your use of
+ * this software. Any use, reproduction, disclosure, or distribution of
+ * this software and related documentation outside the terms of the EULA
+ * is strictly prohibited.
+ *
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <ctype.h>
+#include <assert.h>
+
+#include "cusolverSp.h"
+
+#include "cusolverSp_LOWLEVEL_PREVIEW.h"
+
+#include <cuda_runtime.h>
+
+#include "helper_cuda.h"
+#include "helper_cusolver.h"
+
+template <typename T_ELEM>
+int loadMMSparseMatrix(
+    char *filename,
+    char elem_type,
+    bool csrFormat,
+    int *m,
+    int *n,
+    int *nnz,
+    T_ELEM **aVal,
+    int **aRowInd,
+    int **aColInd,
+    int extendSymMatrix);
+
+void UsageSP(void)
+{
+    printf( "<options>\n");
+    printf( "-h          : display this help\n");
+    printf( "-file=<filename> : filename containing a matrix in MM format\n");
+    printf( "-device=<device_id> : <device_id> if want to run on specific GPU\n");
+
+    exit( 0 );
+}
+
+void parseCommandLineArguments(int argc, char *argv[], struct testOpts &opts)
+{
+    memset(&opts, 0, sizeof(opts));
+
+    if (checkCmdLineFlag(argc, (const char **)argv, "-h"))
+    {
+        UsageSP();
+    }
+
+    if (checkCmdLineFlag(argc, (const char **)argv, "file"))
+    {
+        char *fileName = 0;
+        getCmdLineArgumentString(argc, (const char **)argv, "file", &fileName);
+
+        if (fileName)
+        {
+            opts.sparse_mat_filename = fileName;
+        }
+        else
+        {
+            printf("\nIncorrect filename passed to -file \n ");
+            UsageSP();
+        }
+    }
+}
+
+
+int main (int argc, char *argv[])
+{
+    struct testOpts opts;
+    cusolverSpHandle_t cusolverSpH = NULL; // reordering, permutation and 1st LU factorization
+    cusparseHandle_t   cusparseH = NULL;   // residual evaluation
+    cudaStream_t stream = NULL;
+    cusparseMatDescr_t descrA = NULL; // A is a base-0 general matrix
+
+    csrcholInfoHost_t h_info = NULL; // opaque info structure for LU with parital pivoting
+    csrcholInfo_t d_info = NULL; // opaque info structure for LU with parital pivoting
+
+    int rowsA = 0; // number of rows of A
+    int colsA = 0; // number of columns of A
+    int nnzA  = 0; // number of nonzeros of A
+    int baseA = 0; // base index in CSR format
+
+    // CSR(A) from I/O
+    int *h_csrRowPtrA = NULL; // <int> n+1 
+    int *h_csrColIndA = NULL; // <int> nnzA 
+    double *h_csrValA = NULL; // <double> nnzA 
+
+    double *h_x = NULL; // <double> n,  x = A \ b
+    double *h_b = NULL; // <double> n, b = ones(m,1)
+    double *h_r = NULL; // <double> n, r = b - A*x
+
+    size_t size_internal = 0; 
+    size_t size_chol = 0; // size of working space for csrlu
+    void *buffer_cpu = NULL; // working space for Cholesky
+    void *buffer_gpu = NULL; // working space for Cholesky
+
+    int *d_csrRowPtrA = NULL; // <int> n+1
+    int *d_csrColIndA = NULL; // <int> nnzA
+    double *d_csrValA = NULL; // <double> nnzA
+    double *d_x = NULL; // <double> n, x = A \ b 
+    double *d_b = NULL; // <double> n, a copy of h_b
+    double *d_r = NULL; // <double> n, r = b - A*x
+
+    // the constants used in residual evaluation, r = b - A*x
+    const double minus_one = -1.0;
+    const double one = 1.0;
+    // the constant used in cusolverSp
+    // singularity is -1 if A is invertible under tol
+    // tol determines the condition of singularity
+    int singularity = 0; 
+    const double tol = 1.e-14;
+
+    double x_inf = 0.0; // |x|
+    double r_inf = 0.0; // |r|
+    double A_inf = 0.0; // |A|
+    int errors = 0;
+
+    parseCommandLineArguments(argc, argv, opts);
+
+    findCudaDevice(argc, (const char **)argv);
+
+    if (opts.sparse_mat_filename == NULL)
+    {
+        opts.sparse_mat_filename =  sdkFindFilePath("lap2D_5pt_n100.mtx", argv[0]);
+        if (opts.sparse_mat_filename != NULL)
+            printf("Using default input file [%s]\n", opts.sparse_mat_filename);
+        else
+            printf("Could not find lap2D_5pt_n100.mtx\n");
+    }
+    else
+    {
+        printf("Using input file [%s]\n", opts.sparse_mat_filename);
+    }
+
+    printf("step 1: read matrix market format\n");
+
+    if (opts.sparse_mat_filename)
+    {
+        if (loadMMSparseMatrix<double>(opts.sparse_mat_filename, 'd', true , &rowsA, &colsA,
+               &nnzA, &h_csrValA, &h_csrRowPtrA, &h_csrColIndA, true)) 
+        {
+            return 1;
+        }
+        baseA = h_csrRowPtrA[0]; // baseA = {0,1}
+    }
+    else
+    {
+        fprintf(stderr, "Error: input matrix is not provided\n");
+        return 1;
+    }
+
+    if ( rowsA != colsA ) 
+    {
+        fprintf(stderr, "Error: only support square matrix\n");
+        return 1;
+    }
+
+    printf("sparse matrix A is %d x %d with %d nonzeros, base=%d\n", rowsA, colsA, nnzA, baseA);
+
+    checkCudaErrors(cusolverSpCreate(&cusolverSpH));
+
+    checkCudaErrors(cusparseCreate(&cusparseH));
+
+    checkCudaErrors(cudaStreamCreate(&stream));
+
+    checkCudaErrors(cusolverSpSetStream(cusolverSpH, stream));
+
+    checkCudaErrors(cusparseSetStream(cusparseH, stream));
+
+    checkCudaErrors(cusparseCreateMatDescr(&descrA));
+
+    checkCudaErrors(cusparseSetMatType(descrA, CUSPARSE_MATRIX_TYPE_GENERAL));
+    if (baseA)
+    {
+        checkCudaErrors(cusparseSetMatIndexBase(descrA, CUSPARSE_INDEX_BASE_ONE));
+    }
+    else
+    {
+        checkCudaErrors(cusparseSetMatIndexBase(descrA, CUSPARSE_INDEX_BASE_ZERO));
+    }
+
+    h_x    = (double*)malloc(sizeof(double)*colsA);
+    h_b    = (double*)malloc(sizeof(double)*rowsA);
+    h_r    = (double*)malloc(sizeof(double)*rowsA);
+
+    assert(NULL != h_x);
+    assert(NULL != h_b);
+    assert(NULL != h_r);
+
+    checkCudaErrors(cudaMalloc((void **)&d_csrRowPtrA, sizeof(int)*(rowsA+1)));
+    checkCudaErrors(cudaMalloc((void **)&d_csrColIndA, sizeof(int)*nnzA));
+    checkCudaErrors(cudaMalloc((void **)&d_csrValA   , sizeof(double)*nnzA));
+    checkCudaErrors(cudaMalloc((void **)&d_x, sizeof(double)*colsA));
+    checkCudaErrors(cudaMalloc((void **)&d_b, sizeof(double)*rowsA));
+    checkCudaErrors(cudaMalloc((void **)&d_r, sizeof(double)*rowsA));
+
+    for(int row = 0 ; row < rowsA ; row++)
+    {
+        h_b[row] = 1.0;
+    }
+
+    printf("step 2: create opaque info structure\n");
+    checkCudaErrors(cusolverSpCreateCsrcholInfoHost(&h_info));
+
+    printf("step 3: analyze chol(A) to know structure of L\n");
+    checkCudaErrors(cusolverSpXcsrcholAnalysisHost(
+        cusolverSpH, rowsA, nnzA,
+        descrA, h_csrRowPtrA, h_csrColIndA,
+        h_info));
+
+    printf("step 4: workspace for chol(A)\n");
+    checkCudaErrors(cusolverSpDcsrcholBufferInfoHost(
+        cusolverSpH, rowsA, nnzA,
+        descrA, h_csrValA, h_csrRowPtrA, h_csrColIndA,
+        h_info,
+        &size_internal,
+        &size_chol));
+
+    if (buffer_cpu) 
+    {
+        free(buffer_cpu); 
+    }
+    buffer_cpu = (void*)malloc(sizeof(char)*size_chol);
+    assert(NULL != buffer_cpu);
+
+    printf("step 5: compute A = L*L^T \n");
+    checkCudaErrors(cusolverSpDcsrcholFactorHost(
+        cusolverSpH, rowsA, nnzA,
+        descrA, h_csrValA, h_csrRowPtrA, h_csrColIndA,
+        h_info,
+        buffer_cpu));
+
+    printf("step 6: check if the matrix is singular \n");
+    checkCudaErrors(cusolverSpDcsrcholZeroPivotHost(
+        cusolverSpH, h_info, tol, &singularity));
+
+    if ( 0 <= singularity)
+    {
+        fprintf(stderr, "Error: A is not invertible, singularity=%d\n", singularity);
+        return 1;
+    }
+
+    printf("step 7: solve A*x = b \n");
+    checkCudaErrors(cusolverSpDcsrcholSolveHost(
+        cusolverSpH, rowsA, h_b, h_x, h_info, buffer_cpu));
+
+    printf("step 8: evaluate residual r = b - A*x (result on CPU)\n");
+    // use GPU gemv to compute r = b - A*x
+    checkCudaErrors(cudaMemcpy(d_csrRowPtrA, h_csrRowPtrA, sizeof(int)*(rowsA+1), cudaMemcpyHostToDevice));
+    checkCudaErrors(cudaMemcpy(d_csrColIndA, h_csrColIndA, sizeof(int)*nnzA     , cudaMemcpyHostToDevice));
+    checkCudaErrors(cudaMemcpy(d_csrValA   , h_csrValA   , sizeof(double)*nnzA  , cudaMemcpyHostToDevice));
+
+    checkCudaErrors(cudaMemcpy(d_r, h_b, sizeof(double)*rowsA, cudaMemcpyHostToDevice));
+    checkCudaErrors(cudaMemcpy(d_x, h_x, sizeof(double)*colsA, cudaMemcpyHostToDevice));
+
+    /* Wrap raw data into cuSPARSE generic API objects */
+    cusparseSpMatDescr_t matA = NULL;
+    if (baseA)
+    {
+        checkCudaErrors(cusparseCreateCsr(
+        &matA, rowsA, colsA, nnzA, d_csrRowPtrA, d_csrColIndA, d_csrValA, CUSPARSE_INDEX_32I,
+        CUSPARSE_INDEX_32I, CUSPARSE_INDEX_BASE_ONE, CUDA_R_64F));
+    }
+    else
+    {
+        checkCudaErrors(cusparseCreateCsr(
+        &matA, rowsA, colsA, nnzA, d_csrRowPtrA, d_csrColIndA, d_csrValA, CUSPARSE_INDEX_32I,
+        CUSPARSE_INDEX_32I, CUSPARSE_INDEX_BASE_ZERO, CUDA_R_64F));
+    }
+
+    cusparseDnVecDescr_t vecx = NULL;
+    checkCudaErrors(cusparseCreateDnVec(&vecx, colsA, d_x, CUDA_R_64F));
+    cusparseDnVecDescr_t vecAx = NULL;
+    checkCudaErrors(cusparseCreateDnVec(&vecAx, rowsA, d_r, CUDA_R_64F));
+
+    /* Allocate workspace for cuSPARSE */
+    size_t bufferSize = 0;
+    checkCudaErrors(cusparseSpMV_bufferSize(
+        cusparseH, CUSPARSE_OPERATION_NON_TRANSPOSE, &minus_one, matA, vecx,
+        &one, vecAx, CUDA_R_64F, CUSPARSE_SPMV_ALG_DEFAULT, &bufferSize));
+    void *buffer = NULL;
+    checkCudaErrors(cudaMalloc(&buffer, bufferSize));
+
+    checkCudaErrors(cusparseSpMV(
+        cusparseH, CUSPARSE_OPERATION_NON_TRANSPOSE, &minus_one, matA, vecx,
+        &one, vecAx, CUDA_R_64F, CUSPARSE_SPMV_ALG_DEFAULT, buffer));
+
+    checkCudaErrors(cudaMemcpy(h_r, d_r, sizeof(double)*rowsA, cudaMemcpyDeviceToHost));
+
+    x_inf = vec_norminf(colsA, h_x);
+    r_inf = vec_norminf(rowsA, h_r);
+    A_inf = csr_mat_norminf(rowsA, colsA, nnzA, descrA, h_csrValA, h_csrRowPtrA, h_csrColIndA);
+
+    printf("(CPU) |b - A*x| = %E \n", r_inf);
+    printf("(CPU) |A| = %E \n", A_inf);
+    printf("(CPU) |x| = %E \n", x_inf);
+    printf("(CPU) |b - A*x|/(|A|*|x|) = %E \n", r_inf/(A_inf * x_inf));
+
+    printf("step 9: create opaque info structure\n");
+    checkCudaErrors(cusolverSpCreateCsrcholInfo(&d_info));
+
+    checkCudaErrors(cudaMemcpy(d_csrRowPtrA, h_csrRowPtrA, sizeof(int)*(rowsA+1), cudaMemcpyHostToDevice));
+    checkCudaErrors(cudaMemcpy(d_csrColIndA, h_csrColIndA, sizeof(int)*nnzA     , cudaMemcpyHostToDevice));
+    checkCudaErrors(cudaMemcpy(d_csrValA   , h_csrValA   , sizeof(double)*nnzA  , cudaMemcpyHostToDevice));
+    checkCudaErrors(cudaMemcpy(d_b, h_b, sizeof(double)*rowsA, cudaMemcpyHostToDevice));
+
+    printf("step 10: analyze chol(A) to know structure of L\n");
+    checkCudaErrors(cusolverSpXcsrcholAnalysis(
+        cusolverSpH, rowsA, nnzA,
+        descrA, d_csrRowPtrA, d_csrColIndA,
+        d_info));
+
+    printf("step 11: workspace for chol(A)\n");
+    checkCudaErrors(cusolverSpDcsrcholBufferInfo(
+        cusolverSpH, rowsA, nnzA,
+        descrA, d_csrValA, d_csrRowPtrA, d_csrColIndA,
+        d_info,
+        &size_internal,
+        &size_chol));
+
+    if (buffer_gpu) {
+        checkCudaErrors(cudaFree(buffer_gpu));
+    }
+    checkCudaErrors(cudaMalloc(&buffer_gpu, sizeof(char)*size_chol));
+
+    printf("step 12: compute A = L*L^T \n");
+    checkCudaErrors(cusolverSpDcsrcholFactor(
+        cusolverSpH, rowsA, nnzA,
+        descrA, d_csrValA, d_csrRowPtrA, d_csrColIndA,
+        d_info,
+        buffer_gpu));
+
+    printf("step 13: check if the matrix is singular \n");
+    checkCudaErrors(cusolverSpDcsrcholZeroPivot(
+        cusolverSpH, d_info, tol, &singularity));
+
+    if ( 0 <= singularity){
+        fprintf(stderr, "Error: A is not invertible, singularity=%d\n", singularity);
+        return 1;
+    }
+
+    printf("step 14: solve A*x = b \n");
+    checkCudaErrors(cusolverSpDcsrcholSolve(
+        cusolverSpH, rowsA, d_b, d_x, d_info, buffer_gpu));
+
+    checkCudaErrors(cudaMemcpy(d_r, h_b, sizeof(double)*rowsA, cudaMemcpyHostToDevice));
+
+    checkCudaErrors(cusparseSpMV(
+        cusparseH, CUSPARSE_OPERATION_NON_TRANSPOSE, &minus_one, matA, vecx,
+        &one, vecAx, CUDA_R_64F, CUSPARSE_SPMV_ALG_DEFAULT, buffer));
+
+    checkCudaErrors(cudaMemcpy(h_r, d_r, sizeof(double)*rowsA, cudaMemcpyDeviceToHost));
+
+    r_inf = vec_norminf(rowsA, h_r);
+
+    printf("(GPU) |b - A*x| = %E \n", r_inf);
+    printf("(GPU) |b - A*x|/(|A|*|x|) = %E \n", r_inf/(A_inf * x_inf));
+
+
+    if (cusolverSpH) { checkCudaErrors(cusolverSpDestroy(cusolverSpH)); }
+    if (cusparseH  ) { checkCudaErrors(cusparseDestroy(cusparseH)); }
+    if (stream     ) { checkCudaErrors(cudaStreamDestroy(stream)); }
+    if (descrA     ) { checkCudaErrors(cusparseDestroyMatDescr(descrA)); }
+    if (h_info     ) { checkCudaErrors(cusolverSpDestroyCsrcholInfoHost(h_info)); }
+    if (d_info     ) { checkCudaErrors(cusolverSpDestroyCsrcholInfo(d_info)); }
+    if (matA       ) { checkCudaErrors(cusparseDestroySpMat(matA)); }
+    if (vecx       ) { checkCudaErrors(cusparseDestroyDnVec(vecx)); }
+    if (vecAx      ) { checkCudaErrors(cusparseDestroyDnVec(vecAx)); }
+
+    if (h_csrValA   ) { free(h_csrValA); }
+    if (h_csrRowPtrA) { free(h_csrRowPtrA); }
+    if (h_csrColIndA) { free(h_csrColIndA); }
+
+    if (h_x   ) { free(h_x); }
+    if (h_b   ) { free(h_b); }
+    if (h_r   ) { free(h_r); }
+
+    if (buffer_cpu) { free(buffer_cpu); }
+    if (buffer_gpu) { checkCudaErrors(cudaFree(buffer_gpu)); }
+
+    if (d_csrValA   ) { checkCudaErrors(cudaFree(d_csrValA)); }
+    if (d_csrRowPtrA) { checkCudaErrors(cudaFree(d_csrRowPtrA)); }
+    if (d_csrColIndA) { checkCudaErrors(cudaFree(d_csrColIndA)); }
+    if (d_x) { checkCudaErrors(cudaFree(d_x)); }
+    if (d_b) { checkCudaErrors(cudaFree(d_b)); }
+    if (d_r) { checkCudaErrors(cudaFree(d_r)); }
+
+    return 0;
+}
+

Samples/4_CUDA_Libraries/cuSolverSp_LowlevelCholesky/cuSolverSp_LowlevelCholesky_vs2017.sln

@@ -0,0 +1,20 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio 2017
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "cuSolverSp_LowlevelCholesky", "cuSolverSp_LowlevelCholesky_vs2017.vcxproj", "{997E0757-EA74-4A4E-A0FC-47D8C8831A15}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|x64 = Debug|x64
+		Release|x64 = Release|x64
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Debug|x64.ActiveCfg = Debug|x64
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Debug|x64.Build.0 = Debug|x64
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Release|x64.ActiveCfg = Release|x64
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Release|x64.Build.0 = Release|x64
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+EndGlobal

Samples/4_CUDA_Libraries/cuSolverSp_LowlevelCholesky/cuSolverSp_LowlevelCholesky_vs2017.vcxproj

@@ -0,0 +1,114 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <CUDAPropsPath Condition="'$(CUDAPropsPath)'==''">$(VCTargetsPath)\BuildCustomizations</CUDAPropsPath>
+  </PropertyGroup>
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Debug|x64">
+      <Configuration>Debug</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|x64">
+      <Configuration>Release</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>{997E0757-EA74-4A4E-A0FC-47D8C8831A15}</ProjectGuid>
+    <RootNamespace>cuSolverSp_LowlevelCholesky_vs2017</RootNamespace>
+    <ProjectName>cuSolverSp_LowlevelCholesky</ProjectName>
+    <CudaToolkitCustomDir />
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(WindowsTargetPlatformVersion)'==''">
+    <LatestTargetPlatformVersion>$([Microsoft.Build.Utilities.ToolLocationHelper]::GetLatestSDKTargetPlatformVersion('Windows', '10.0'))</LatestTargetPlatformVersion>
+    <WindowsTargetPlatformVersion Condition="'$(WindowsTargetPlatformVersion)' == ''">$(LatestTargetPlatformVersion)</WindowsTargetPlatformVersion>
+    <TargetPlatformVersion>$(WindowsTargetPlatformVersion)</TargetPlatformVersion>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <PropertyGroup>
+    <ConfigurationType>Application</ConfigurationType>
+    <CharacterSet>MultiByte</CharacterSet>
+    <PlatformToolset>v141</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)'=='Debug'">
+    <UseDebugLibraries>true</UseDebugLibraries>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)'=='Release'">
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <ImportGroup Label="ExtensionSettings">
+    <Import Project="$(CUDAPropsPath)\CUDA 11.6.props" />
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets">
+    <Import Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" />
+  </ImportGroup>
+  <PropertyGroup Label="UserMacros" />
+  <PropertyGroup>
+    <IntDir>$(Platform)/$(Configuration)/</IntDir>
+    <IncludePath>$(IncludePath)</IncludePath>
+    <CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRules />
+    <CodeAnalysisRuleAssemblies />
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Platform)'=='x64'">
+    <OutDir>../../../bin/win64/$(Configuration)/</OutDir>
+  </PropertyGroup>
+  <ItemDefinitionGroup>
+    <ClCompile>
+      <WarningLevel>Level3</WarningLevel>
+      <PreprocessorDefinitions>WIN32;_MBCS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <AdditionalIncludeDirectories>./;$(CudaToolkitDir)/include;../../../Common;$(CudaToolkitIncludeDir);</AdditionalIncludeDirectories>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <AdditionalDependencies>cusolver.lib;cusparse.lib;cudart_static.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalLibraryDirectories>$(CudaToolkitLibDir);</AdditionalLibraryDirectories>
+      <OutputFile>$(OutDir)/cuSolverSp_LowlevelCholesky.exe</OutputFile>
+    </Link>
+    <CudaCompile>
+      <CodeGeneration>compute_35,sm_35;compute_37,sm_37;compute_50,sm_50;compute_52,sm_52;compute_60,sm_60;compute_61,sm_61;compute_70,sm_70;compute_75,sm_75;compute_80,sm_80;compute_86,sm_86;</CodeGeneration>
+      <AdditionalOptions>-Xcompiler "/wd 4819"  --threads 0 </AdditionalOptions>
+      <Include>./;../../../Common</Include>
+      <Defines>WIN32</Defines>
+    </CudaCompile>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)'=='Debug'">
+    <ClCompile>
+      <Optimization>Disabled</Optimization>
+      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <LinkTimeCodeGeneration>Default</LinkTimeCodeGeneration>
+    </Link>
+    <CudaCompile>
+      <Runtime>MTd</Runtime>
+      <TargetMachinePlatform>64</TargetMachinePlatform>
+    </CudaCompile>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)'=='Release'">
+    <ClCompile>
+      <Optimization>MaxSpeed</Optimization>
+      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <GenerateDebugInformation>false</GenerateDebugInformation>
+      <LinkTimeCodeGeneration>UseLinkTimeCodeGeneration</LinkTimeCodeGeneration>
+    </Link>
+    <CudaCompile>
+      <Runtime>MT</Runtime>
+      <TargetMachinePlatform>64</TargetMachinePlatform>
+    </CudaCompile>
+  </ItemDefinitionGroup>
+  <ItemGroup>
+    <ClCompile Include="cuSolverSp_LowlevelCholesky.cpp" />
+    <ClCompile Include="mmio.c" />
+    <ClCompile Include="mmio_wrapper.cpp" />
+    <ClInclude Include="mmio.h" />
+  </ItemGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <ImportGroup Label="ExtensionTargets">
+    <Import Project="$(CUDAPropsPath)\CUDA 11.6.targets" />
+  </ImportGroup>
+</Project>

Samples/4_CUDA_Libraries/cuSolverSp_LowlevelCholesky/cuSolverSp_LowlevelCholesky_vs2019.sln

@@ -0,0 +1,20 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio 2019
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "cuSolverSp_LowlevelCholesky", "cuSolverSp_LowlevelCholesky_vs2019.vcxproj", "{997E0757-EA74-4A4E-A0FC-47D8C8831A15}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|x64 = Debug|x64
+		Release|x64 = Release|x64
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Debug|x64.ActiveCfg = Debug|x64
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Debug|x64.Build.0 = Debug|x64
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Release|x64.ActiveCfg = Release|x64
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Release|x64.Build.0 = Release|x64
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+EndGlobal

Samples/4_CUDA_Libraries/cuSolverSp_LowlevelCholesky/cuSolverSp_LowlevelCholesky_vs2019.vcxproj

@@ -0,0 +1,110 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <CUDAPropsPath Condition="'$(CUDAPropsPath)'==''">$(VCTargetsPath)\BuildCustomizations</CUDAPropsPath>
+  </PropertyGroup>
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Debug|x64">
+      <Configuration>Debug</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|x64">
+      <Configuration>Release</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>{997E0757-EA74-4A4E-A0FC-47D8C8831A15}</ProjectGuid>
+    <RootNamespace>cuSolverSp_LowlevelCholesky_vs2019</RootNamespace>
+    <ProjectName>cuSolverSp_LowlevelCholesky</ProjectName>
+    <CudaToolkitCustomDir />
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <PropertyGroup>
+    <ConfigurationType>Application</ConfigurationType>
+    <CharacterSet>MultiByte</CharacterSet>
+    <PlatformToolset>v142</PlatformToolset>
+    <WindowsTargetPlatformVersion>10.0</WindowsTargetPlatformVersion>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)'=='Debug'">
+    <UseDebugLibraries>true</UseDebugLibraries>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)'=='Release'">
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <ImportGroup Label="ExtensionSettings">
+    <Import Project="$(CUDAPropsPath)\CUDA 11.6.props" />
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets">
+    <Import Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" />
+  </ImportGroup>
+  <PropertyGroup Label="UserMacros" />
+  <PropertyGroup>
+    <IntDir>$(Platform)/$(Configuration)/</IntDir>
+    <IncludePath>$(IncludePath)</IncludePath>
+    <CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRules />
+    <CodeAnalysisRuleAssemblies />
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Platform)'=='x64'">
+    <OutDir>../../../bin/win64/$(Configuration)/</OutDir>
+  </PropertyGroup>
+  <ItemDefinitionGroup>
+    <ClCompile>
+      <WarningLevel>Level3</WarningLevel>
+      <PreprocessorDefinitions>WIN32;_MBCS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <AdditionalIncludeDirectories>./;$(CudaToolkitDir)/include;../../../Common;$(CudaToolkitIncludeDir);</AdditionalIncludeDirectories>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <AdditionalDependencies>cusolver.lib;cusparse.lib;cudart_static.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalLibraryDirectories>$(CudaToolkitLibDir);</AdditionalLibraryDirectories>
+      <OutputFile>$(OutDir)/cuSolverSp_LowlevelCholesky.exe</OutputFile>
+    </Link>
+    <CudaCompile>
+      <CodeGeneration>compute_35,sm_35;compute_37,sm_37;compute_50,sm_50;compute_52,sm_52;compute_60,sm_60;compute_61,sm_61;compute_70,sm_70;compute_75,sm_75;compute_80,sm_80;compute_86,sm_86;</CodeGeneration>
+      <AdditionalOptions>-Xcompiler "/wd 4819"  --threads 0 </AdditionalOptions>
+      <Include>./;../../../Common</Include>
+      <Defines>WIN32</Defines>
+    </CudaCompile>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)'=='Debug'">
+    <ClCompile>
+      <Optimization>Disabled</Optimization>
+      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <LinkTimeCodeGeneration>Default</LinkTimeCodeGeneration>
+    </Link>
+    <CudaCompile>
+      <Runtime>MTd</Runtime>
+      <TargetMachinePlatform>64</TargetMachinePlatform>
+    </CudaCompile>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)'=='Release'">
+    <ClCompile>
+      <Optimization>MaxSpeed</Optimization>
+      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <GenerateDebugInformation>false</GenerateDebugInformation>
+      <LinkTimeCodeGeneration>UseLinkTimeCodeGeneration</LinkTimeCodeGeneration>
+    </Link>
+    <CudaCompile>
+      <Runtime>MT</Runtime>
+      <TargetMachinePlatform>64</TargetMachinePlatform>
+    </CudaCompile>
+  </ItemDefinitionGroup>
+  <ItemGroup>
+    <ClCompile Include="cuSolverSp_LowlevelCholesky.cpp" />
+    <ClCompile Include="mmio.c" />
+    <ClCompile Include="mmio_wrapper.cpp" />
+    <ClInclude Include="mmio.h" />
+  </ItemGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <ImportGroup Label="ExtensionTargets">
+    <Import Project="$(CUDAPropsPath)\CUDA 11.6.targets" />
+  </ImportGroup>
+</Project>

Samples/4_CUDA_Libraries/cuSolverSp_LowlevelCholesky/cuSolverSp_LowlevelCholesky_vs2022.sln

@@ -0,0 +1,20 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio 2022
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "cuSolverSp_LowlevelCholesky", "cuSolverSp_LowlevelCholesky_vs2022.vcxproj", "{997E0757-EA74-4A4E-A0FC-47D8C8831A15}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|x64 = Debug|x64
+		Release|x64 = Release|x64
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Debug|x64.ActiveCfg = Debug|x64
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Debug|x64.Build.0 = Debug|x64
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Release|x64.ActiveCfg = Release|x64
+		{997E0757-EA74-4A4E-A0FC-47D8C8831A15}.Release|x64.Build.0 = Release|x64
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+EndGlobal

Samples/4_CUDA_Libraries/cuSolverSp_LowlevelCholesky/cuSolverSp_LowlevelCholesky_vs2022.vcxproj

@@ -0,0 +1,110 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <CUDAPropsPath Condition="'$(CUDAPropsPath)'==''">$(VCTargetsPath)\BuildCustomizations</CUDAPropsPath>
+  </PropertyGroup>
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Debug|x64">
+      <Configuration>Debug</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|x64">
+      <Configuration>Release</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>{997E0757-EA74-4A4E-A0FC-47D8C8831A15}</ProjectGuid>
+    <RootNamespace>cuSolverSp_LowlevelCholesky_vs2022</RootNamespace>
+    <ProjectName>cuSolverSp_LowlevelCholesky</ProjectName>
+    <CudaToolkitCustomDir />
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <PropertyGroup>
+    <ConfigurationType>Application</ConfigurationType>
+    <CharacterSet>MultiByte</CharacterSet>
+    <PlatformToolset>v143</PlatformToolset>
+    <WindowsTargetPlatformVersion>10.0</WindowsTargetPlatformVersion>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)'=='Debug'">
+    <UseDebugLibraries>true</UseDebugLibraries>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)'=='Release'">
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <ImportGroup Label="ExtensionSettings">
+    <Import Project="$(CUDAPropsPath)\CUDA 11.6.props" />
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets">
+    <Import Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" />
+  </ImportGroup>
+  <PropertyGroup Label="UserMacros" />
+  <PropertyGroup>
+    <IntDir>$(Platform)/$(Configuration)/</IntDir>
+    <IncludePath>$(IncludePath)</IncludePath>
+    <CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRules />
+    <CodeAnalysisRuleAssemblies />
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Platform)'=='x64'">
+    <OutDir>../../../bin/win64/$(Configuration)/</OutDir>
+  </PropertyGroup>
+  <ItemDefinitionGroup>
+    <ClCompile>
+      <WarningLevel>Level3</WarningLevel>
+      <PreprocessorDefinitions>WIN32;_MBCS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <AdditionalIncludeDirectories>./;$(CudaToolkitDir)/include;../../../Common;$(CudaToolkitIncludeDir);</AdditionalIncludeDirectories>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <AdditionalDependencies>cusolver.lib;cusparse.lib;cudart_static.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalLibraryDirectories>$(CudaToolkitLibDir);</AdditionalLibraryDirectories>
+      <OutputFile>$(OutDir)/cuSolverSp_LowlevelCholesky.exe</OutputFile>
+    </Link>
+    <CudaCompile>
+      <CodeGeneration>compute_35,sm_35;compute_37,sm_37;compute_50,sm_50;compute_52,sm_52;compute_60,sm_60;compute_61,sm_61;compute_70,sm_70;compute_75,sm_75;compute_80,sm_80;compute_86,sm_86;</CodeGeneration>
+      <AdditionalOptions>-Xcompiler "/wd 4819"  --threads 0 </AdditionalOptions>
+      <Include>./;../../../Common</Include>
+      <Defines>WIN32</Defines>
+    </CudaCompile>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)'=='Debug'">
+    <ClCompile>
+      <Optimization>Disabled</Optimization>
+      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <LinkTimeCodeGeneration>Default</LinkTimeCodeGeneration>
+    </Link>
+    <CudaCompile>
+      <Runtime>MTd</Runtime>
+      <TargetMachinePlatform>64</TargetMachinePlatform>
+    </CudaCompile>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)'=='Release'">
+    <ClCompile>
+      <Optimization>MaxSpeed</Optimization>
+      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
+    </ClCompile>
+    <Link>
+      <GenerateDebugInformation>false</GenerateDebugInformation>
+      <LinkTimeCodeGeneration>UseLinkTimeCodeGeneration</LinkTimeCodeGeneration>
+    </Link>
+    <CudaCompile>
+      <Runtime>MT</Runtime>
+      <TargetMachinePlatform>64</TargetMachinePlatform>
+    </CudaCompile>
+  </ItemDefinitionGroup>
+  <ItemGroup>
+    <ClCompile Include="cuSolverSp_LowlevelCholesky.cpp" />
+    <ClCompile Include="mmio.c" />
+    <ClCompile Include="mmio_wrapper.cpp" />
+    <ClInclude Include="mmio.h" />
+  </ItemGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <ImportGroup Label="ExtensionTargets">
+    <Import Project="$(CUDAPropsPath)\CUDA 11.6.targets" />
+  </ImportGroup>
+</Project>

Samples/4_CUDA_Libraries/cuSolverSp_LowlevelCholesky/mmio.c

@@ -0,0 +1,517 @@
+/* 
+*   Matrix Market I/O library for ANSI C
+*
+*   See http://math.nist.gov/MatrixMarket for details.
+*
+*
+*/
+
+/* avoid Windows warnings (for example: strcpy, fscanf, etc.) */
+#if defined(_WIN32)  
+#define _CRT_SECURE_NO_WARNINGS
+#endif
+
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include <ctype.h>
+
+#include "mmio.h"
+
+int mm_read_unsymmetric_sparse(const char *fname, int *M_, int *N_, int *nz_,
+                double **val_, int **I_, int **J_)
+{
+    FILE *f;
+    MM_typecode matcode;
+    int M, N, nz;
+    int i;
+    double *val;
+    int *I, *J;
+ 
+    if ((f = fopen(fname, "r")) == NULL)
+            return -1;
+ 
+ 
+    if (mm_read_banner(f, &matcode) != 0)
+    {
+        printf("mm_read_unsymetric: Could not process Matrix Market banner ");
+        printf(" in file [%s]\n", fname);
+        return -1;
+    }
+ 
+ 
+ 
+    if ( !(mm_is_real(matcode) && mm_is_matrix(matcode) &&
+            mm_is_sparse(matcode)))
+    {
+        fprintf(stderr, "Sorry, this application does not support ");
+        fprintf(stderr, "Market Market type: [%s]\n",
+                mm_typecode_to_str(matcode));
+        return -1;
+    }
+ 
+    /* find out size of sparse matrix: M, N, nz .... */
+ 
+    if (mm_read_mtx_crd_size(f, &M, &N, &nz) !=0)
+    {
+        fprintf(stderr, "read_unsymmetric_sparse(): could not parse matrix size.\n");
+        return -1;
+    }
+ 
+    *M_ = M;
+    *N_ = N;
+    *nz_ = nz;
+ 
+    /* reserve memory for matrices */
+ 
+    I = (int *) malloc(nz * sizeof(int));
+    J = (int *) malloc(nz * sizeof(int));
+    val = (double *) malloc(nz * sizeof(double));
+ 
+    *val_ = val;
+    *I_ = I;
+    *J_ = J;
+ 
+    /* NOTE: when reading in doubles, ANSI C requires the use of the "l"  */
+    /*   specifier as in "%lg", "%lf", "%le", otherwise errors will occur */
+    /*  (ANSI C X3.159-1989, Sec. 4.9.6.2, p. 136 lines 13-15)            */
+ 
+    for (i=0; i<nz; i++)
+    {
+        if (fscanf(f, "%d %d %lg\n", &I[i], &J[i], &val[i]) != 3) {
+            return -1;
+        }
+        I[i]--;  /* adjust from 1-based to 0-based */
+        J[i]--;
+    }
+    fclose(f);
+ 
+    return 0;
+}
+
+int mm_is_valid(MM_typecode matcode)
+{
+    if (!mm_is_matrix(matcode)) return 0;
+    if (mm_is_dense(matcode) && mm_is_pattern(matcode)) return 0;
+    if (mm_is_real(matcode) && mm_is_hermitian(matcode)) return 0;
+    if (mm_is_pattern(matcode) && (mm_is_hermitian(matcode) || 
+                mm_is_skew(matcode))) return 0;
+    return 1;
+}
+
+int mm_read_banner(FILE *f, MM_typecode *matcode)
+{
+    char line[MM_MAX_LINE_LENGTH];
+    char banner[MM_MAX_TOKEN_LENGTH];
+    char mtx[MM_MAX_TOKEN_LENGTH]; 
+    char crd[MM_MAX_TOKEN_LENGTH];
+    char data_type[MM_MAX_TOKEN_LENGTH];
+    char storage_scheme[MM_MAX_TOKEN_LENGTH];
+    char *p;
+
+
+    mm_clear_typecode(matcode);  
+
+    if (fgets(line, MM_MAX_LINE_LENGTH, f) == NULL) 
+        return MM_PREMATURE_EOF;
+
+    if (sscanf(line, "%s %s %s %s %s", banner, mtx, crd, data_type, 
+        storage_scheme) != 5)
+        return MM_PREMATURE_EOF;
+
+    for (p=mtx; *p!='\0'; *p=tolower(*p),p++);  /* convert to lower case */
+    for (p=crd; *p!='\0'; *p=tolower(*p),p++);  
+    for (p=data_type; *p!='\0'; *p=tolower(*p),p++);
+    for (p=storage_scheme; *p!='\0'; *p=tolower(*p),p++);
+
+    /* check for banner */
+    if (strncmp(banner, MatrixMarketBanner, strlen(MatrixMarketBanner)) != 0)
+        return MM_NO_HEADER;
+
+    /* first field should be "mtx" */
+    if (strcmp(mtx, MM_MTX_STR) != 0)
+        return  MM_UNSUPPORTED_TYPE;
+    mm_set_matrix(matcode);
+
+
+    /* second field describes whether this is a sparse matrix (in coordinate
+            storage) or a dense array */
+
+
+    if (strcmp(crd, MM_SPARSE_STR) == 0)
+        mm_set_sparse(matcode);
+    else
+    if (strcmp(crd, MM_DENSE_STR) == 0)
+            mm_set_dense(matcode);
+    else
+        return MM_UNSUPPORTED_TYPE;
+    
+
+    /* third field */
+
+    if (strcmp(data_type, MM_REAL_STR) == 0)
+        mm_set_real(matcode);
+    else
+    if (strcmp(data_type, MM_COMPLEX_STR) == 0)
+        mm_set_complex(matcode);
+    else
+    if (strcmp(data_type, MM_PATTERN_STR) == 0)
+        mm_set_pattern(matcode);
+    else
+    if (strcmp(data_type, MM_INT_STR) == 0)
+        mm_set_integer(matcode);
+    else
+        return MM_UNSUPPORTED_TYPE;
+    
+
+    /* fourth field */
+
+    if (strcmp(storage_scheme, MM_GENERAL_STR) == 0)
+        mm_set_general(matcode);
+    else
+    if (strcmp(storage_scheme, MM_SYMM_STR) == 0)
+        mm_set_symmetric(matcode);
+    else
+    if (strcmp(storage_scheme, MM_HERM_STR) == 0)
+        mm_set_hermitian(matcode);
+    else
+    if (strcmp(storage_scheme, MM_SKEW_STR) == 0)
+        mm_set_skew(matcode);
+    else
+        return MM_UNSUPPORTED_TYPE;
+        
+
+    return 0;
+}
+
+int mm_write_mtx_crd_size(FILE *f, int M, int N, int nz)
+{
+    if (fprintf(f, "%d %d %d\n", M, N, nz) != 3)
+        return MM_COULD_NOT_WRITE_FILE;
+    else 
+        return 0;
+}
+
+int mm_read_mtx_crd_size(FILE *f, int *M, int *N, int *nz )
+{
+    char line[MM_MAX_LINE_LENGTH];
+    int num_items_read;
+
+    /* set return null parameter values, in case we exit with errors */
+    *M = *N = *nz = 0;
+
+    /* now continue scanning until you reach the end-of-comments */
+    do 
+    {
+        if (fgets(line,MM_MAX_LINE_LENGTH,f) == NULL) 
+            return MM_PREMATURE_EOF;
+    }while (line[0] == '%');
+
+    /* line[] is either blank or has M,N, nz */
+    if (sscanf(line, "%d %d %d", M, N, nz) == 3)
+        return 0;
+
+    else
+    do
+    {
+        num_items_read = fscanf(f, "%d %d %d", M, N, nz); 
+        if (num_items_read == EOF) return MM_PREMATURE_EOF;
+    }
+    while (num_items_read != 3);
+
+    return 0;
+}
+
+
+int mm_read_mtx_array_size(FILE *f, int *M, int *N)
+{
+    char line[MM_MAX_LINE_LENGTH];
+    int num_items_read;
+    /* set return null parameter values, in case we exit with errors */
+    *M = *N = 0;
+
+    /* now continue scanning until you reach the end-of-comments */
+    do
+    {
+        if (fgets(line,MM_MAX_LINE_LENGTH,f) == NULL)
+            return MM_PREMATURE_EOF;
+    }while (line[0] == '%');
+
+    /* line[] is either blank or has M,N, nz */
+    if (sscanf(line, "%d %d", M, N) == 2)
+        return 0;
+
+    else /* we have a blank line */
+    do
+    {
+        num_items_read = fscanf(f, "%d %d", M, N);
+        if (num_items_read == EOF) return MM_PREMATURE_EOF;
+    }
+    while (num_items_read != 2);
+
+    return 0;
+}
+
+int mm_write_mtx_array_size(FILE *f, int M, int N)
+{
+    if (fprintf(f, "%d %d\n", M, N) != 2)
+        return MM_COULD_NOT_WRITE_FILE;
+    else 
+        return 0;
+}
+
+
+
+/*-------------------------------------------------------------------------*/
+
+/******************************************************************/
+/* use when I[], J[], and val[]J, and val[] are already allocated */
+/******************************************************************/
+
+int mm_read_mtx_crd_data(FILE *f, int M, int N, int nz, int I[], int J[],
+        double val[], MM_typecode matcode)
+{
+    int i;
+    if (mm_is_complex(matcode))
+    {
+        for (i=0; i<nz; i++)
+            if (fscanf(f, "%d %d %lg %lg", &I[i], &J[i], &val[2*i], &val[2*i+1])
+                != 4) return MM_PREMATURE_EOF;
+    }
+    else if (mm_is_real(matcode) || mm_is_integer(matcode))
+    {
+        for (i=0; i<nz; i++)
+        {
+            if (fscanf(f, "%d %d %lg\n", &I[i], &J[i], &val[i])
+                != 3) return MM_PREMATURE_EOF;
+
+        }
+    }
+
+    else if (mm_is_pattern(matcode))
+    {
+        for (i=0; i<nz; i++)
+            if (fscanf(f, "%d %d", &I[i], &J[i])
+                != 2) return MM_PREMATURE_EOF;
+    }
+    else
+        return MM_UNSUPPORTED_TYPE;
+
+    return 0;
+}
+
+int mm_read_mtx_crd_entry(FILE *f, int *I, int *J,
+        double *real, double *imag, MM_typecode matcode)
+{
+    if (mm_is_complex(matcode))
+    {
+            if (fscanf(f, "%d %d %lg %lg", I, J, real, imag)
+                != 4) return MM_PREMATURE_EOF;
+    }
+    else if (mm_is_real(matcode) || mm_is_integer(matcode))
+    {
+            if (fscanf(f, "%d %d %lg\n", I, J, real)
+                != 3) return MM_PREMATURE_EOF;
+
+    }
+
+    else if (mm_is_pattern(matcode))
+    {
+            if (fscanf(f, "%d %d", I, J) != 2) return MM_PREMATURE_EOF;
+    }
+    else
+        return MM_UNSUPPORTED_TYPE;
+
+    return 0;
+}
+
+
+/************************************************************************
+    mm_read_mtx_crd()  fills M, N, nz, array of values, and return
+                        type code, e.g. 'MCRS'
+
+                        if matrix is complex, values[] is of size 2*nz,
+                            (nz pairs of real/imaginary values)
+************************************************************************/
+
+int mm_read_mtx_crd(char *fname, int *M, int *N, int *nz, int **I, int **J, 
+        double **val, MM_typecode *matcode)
+{
+    int ret_code;
+    FILE *f;
+
+    if (strcmp(fname, "stdin") == 0) f=stdin;
+    else
+    if ((f = fopen(fname, "r")) == NULL)
+        return MM_COULD_NOT_READ_FILE;
+
+
+    if ((ret_code = mm_read_banner(f, matcode)) != 0)
+        return ret_code;
+
+    if (!(mm_is_valid(*matcode) && mm_is_sparse(*matcode) && 
+            mm_is_matrix(*matcode)))
+        return MM_UNSUPPORTED_TYPE;
+
+    if ((ret_code = mm_read_mtx_crd_size(f, M, N, nz)) != 0)
+        return ret_code;
+
+
+    *I = (int *)  malloc(*nz * sizeof(int));
+    *J = (int *)  malloc(*nz * sizeof(int));
+    *val = NULL;
+
+    if (mm_is_complex(*matcode))
+    {
+        *val = (double *) malloc(*nz * 2 * sizeof(double));
+        ret_code = mm_read_mtx_crd_data(f, *M, *N, *nz, *I, *J, *val, 
+                *matcode);
+        if (ret_code != 0) return ret_code;
+    }
+    else if (mm_is_real(*matcode) || mm_is_integer(*matcode))
+    {
+        *val = (double *) malloc(*nz * sizeof(double));
+        ret_code = mm_read_mtx_crd_data(f, *M, *N, *nz, *I, *J, *val, 
+                *matcode);
+        if (ret_code != 0) return ret_code;
+    }
+
+    else if (mm_is_pattern(*matcode))
+    {
+        ret_code = mm_read_mtx_crd_data(f, *M, *N, *nz, *I, *J, *val, 
+                *matcode);
+        if (ret_code != 0) return ret_code;
+    }
+
+    if (f != stdin) fclose(f);
+    return 0;
+}
+
+int mm_write_banner(FILE *f, MM_typecode matcode)
+{
+    char *str = mm_typecode_to_str(matcode);
+    int ret_code;
+
+    ret_code = fprintf(f, "%s %s\n", MatrixMarketBanner, str);
+    free(str);
+    if (ret_code !=2 )
+        return MM_COULD_NOT_WRITE_FILE;
+    else
+        return 0;
+}
+
+int mm_write_mtx_crd(char fname[], int M, int N, int nz, int I[], int J[],
+        double val[], MM_typecode matcode)
+{
+    FILE *f;
+    int i;
+
+    if (strcmp(fname, "stdout") == 0)
+        f = stdout;
+    else
+    if ((f = fopen(fname, "w")) == NULL)
+        return MM_COULD_NOT_WRITE_FILE;
+
+    /* print banner followed by typecode */
+    fprintf(f, "%s ", MatrixMarketBanner);
+    fprintf(f, "%s\n", mm_typecode_to_str(matcode));
+
+    /* print matrix sizes and nonzeros */
+    fprintf(f, "%d %d %d\n", M, N, nz);
+
+    /* print values */
+    if (mm_is_pattern(matcode))
+        for (i=0; i<nz; i++)
+            fprintf(f, "%d %d\n", I[i], J[i]);
+    else
+    if (mm_is_integer(matcode))
+        for (i=0; i<nz; i++)
+            fprintf(f, "%d %d %d\n", I[i], J[i], (int)val[i]);
+    else
+    if (mm_is_real(matcode))
+        for (i=0; i<nz; i++)
+            fprintf(f, "%d %d %20.16g\n", I[i], J[i], val[i]);
+    else
+    if (mm_is_complex(matcode))
+        for (i=0; i<nz; i++)
+            fprintf(f, "%d %d %20.16g %20.16g\n", I[i], J[i], val[2*i], 
+                        val[2*i+1]);
+    else
+    {
+        if (f != stdout) fclose(f);
+        return MM_UNSUPPORTED_TYPE;
+    }
+
+    if (f !=stdout) fclose(f);
+
+    return 0;
+}
+
+/**
+*  Create a new copy of a string s.  mm_strdup() is a common routine, but
+*  not part of ANSI C, so it is included here.  Used by mm_typecode_to_str().
+*
+*/
+static char *mm_strdup(const char *s)
+{
+    size_t len = strlen(s);
+    char *s2 = (char *) malloc((len+1)*sizeof(char));
+    return strcpy(s2, s);
+}
+
+char  *mm_typecode_to_str(MM_typecode matcode)
+{
+    char buffer[MM_MAX_LINE_LENGTH];
+    char *types[4];
+    //char *mm_strdup(const char *);
+    //int error =0;
+
+    /* check for MTX type */
+    if (mm_is_matrix(matcode))
+        types[0] = MM_MTX_STR;
+    else
+        return NULL; //  error=1;
+
+    /* check for CRD or ARR matrix */
+    if (mm_is_sparse(matcode))
+        types[1] = MM_SPARSE_STR;
+    else
+    if (mm_is_dense(matcode))
+        types[1] = MM_DENSE_STR;
+    else
+        return NULL;
+
+    /* check for element data type */
+    if (mm_is_real(matcode))
+        types[2] = MM_REAL_STR;
+    else
+    if (mm_is_complex(matcode))
+        types[2] = MM_COMPLEX_STR;
+    else
+    if (mm_is_pattern(matcode))
+        types[2] = MM_PATTERN_STR;
+    else
+    if (mm_is_integer(matcode))
+        types[2] = MM_INT_STR;
+    else
+        return NULL;
+
+
+    /* check for symmetry type */
+    if (mm_is_general(matcode))
+        types[3] = MM_GENERAL_STR;
+    else
+    if (mm_is_symmetric(matcode))
+        types[3] = MM_SYMM_STR;
+    else
+    if (mm_is_hermitian(matcode))
+        types[3] = MM_HERM_STR;
+    else
+    if (mm_is_skew(matcode))
+        types[3] = MM_SKEW_STR;
+    else
+        return NULL;
+
+    sprintf(buffer,"%s %s %s %s", types[0], types[1], types[2], types[3]);
+    return mm_strdup(buffer);
+}

Samples/4_CUDA_Libraries/cuSolverSp_LowlevelCholesky/mmio.h

@@ -0,0 +1,141 @@
+/* 
+*   Matrix Market I/O library for ANSI C
+*
+*   See http://math.nist.gov/MatrixMarket for details.
+*
+*
+*/
+
+#ifndef MM_IO_H
+#define MM_IO_H
+
+#if defined(__cplusplus)
+extern "C" {
+#endif /* __cplusplus */
+
+#define MM_MAX_LINE_LENGTH 1025
+#define MatrixMarketBanner "%%MatrixMarket"
+#define MM_MAX_TOKEN_LENGTH 64
+
+typedef char MM_typecode[4];
+
+char *mm_typecode_to_str(MM_typecode matcode);
+
+int mm_read_banner(FILE *f, MM_typecode *matcode);
+int mm_read_mtx_crd_size(FILE *f, int *M, int *N, int *nz);
+int mm_read_mtx_array_size(FILE *f, int *M, int *N);
+
+int mm_write_banner(FILE *f, MM_typecode matcode);
+int mm_write_mtx_crd_size(FILE *f, int M, int N, int nz);
+int mm_write_mtx_array_size(FILE *f, int M, int N);
+
+
+/********************* MM_typecode query fucntions ***************************/
+
+#define mm_is_matrix(typecode)	((typecode)[0]=='M')
+
+#define mm_is_sparse(typecode)	((typecode)[1]=='C')
+#define mm_is_coordinate(typecode)((typecode)[1]=='C')
+#define mm_is_dense(typecode)	((typecode)[1]=='A')
+#define mm_is_array(typecode)	((typecode)[1]=='A')
+
+#define mm_is_complex(typecode)	((typecode)[2]=='C')
+#define mm_is_real(typecode)		((typecode)[2]=='R')
+#define mm_is_pattern(typecode)	((typecode)[2]=='P')
+#define mm_is_integer(typecode) ((typecode)[2]=='I')
+
+#define mm_is_symmetric(typecode)((typecode)[3]=='S')
+#define mm_is_general(typecode)	((typecode)[3]=='G')
+#define mm_is_skew(typecode)	((typecode)[3]=='K')
+#define mm_is_hermitian(typecode)((typecode)[3]=='H')
+
+int mm_is_valid(MM_typecode matcode);		/* too complex for a macro */
+
+
+/********************* MM_typecode modify fucntions ***************************/
+
+#define mm_set_matrix(typecode)	((*typecode)[0]='M')
+#define mm_set_coordinate(typecode)	((*typecode)[1]='C')
+#define mm_set_array(typecode)	((*typecode)[1]='A')
+#define mm_set_dense(typecode)	mm_set_array(typecode)
+#define mm_set_sparse(typecode)	mm_set_coordinate(typecode)
+
+#define mm_set_complex(typecode)((*typecode)[2]='C')
+#define mm_set_real(typecode)	((*typecode)[2]='R')
+#define mm_set_pattern(typecode)((*typecode)[2]='P')
+#define mm_set_integer(typecode)((*typecode)[2]='I')
+
+
+#define mm_set_symmetric(typecode)((*typecode)[3]='S')
+#define mm_set_general(typecode)((*typecode)[3]='G')
+#define mm_set_skew(typecode)	((*typecode)[3]='K')
+#define mm_set_hermitian(typecode)((*typecode)[3]='H')
+
+#define mm_clear_typecode(typecode) ((*typecode)[0]=(*typecode)[1]= \
+									(*typecode)[2]=' ',(*typecode)[3]='G')
+
+#define mm_initialize_typecode(typecode) mm_clear_typecode(typecode)
+
+
+/********************* Matrix Market error codes ***************************/
+
+
+#define MM_COULD_NOT_READ_FILE	11
+#define MM_PREMATURE_EOF		12
+#define MM_NOT_MTX				13
+#define MM_NO_HEADER			14
+#define MM_UNSUPPORTED_TYPE		15
+#define MM_LINE_TOO_LONG		16
+#define MM_COULD_NOT_WRITE_FILE	17
+
+
+/******************** Matrix Market internal definitions ********************
+
+   MM_matrix_typecode: 4-character sequence
+
+				    ojbect 		sparse/   	data        storage 
+						  		dense     	type        scheme
+
+   string position:	 [0]        [1]			[2]         [3]
+
+   Matrix typecode:  M(atrix)  C(oord)		R(eal)   	G(eneral)
+						        A(array)	C(omplex)   H(ermitian)
+											P(attern)   S(ymmetric)
+								    		I(nteger)	K(kew)
+
+ ***********************************************************************/
+
+#define MM_MTX_STR		"matrix"
+#define MM_ARRAY_STR	"array"
+#define MM_DENSE_STR	"array"
+#define MM_COORDINATE_STR "coordinate" 
+#define MM_SPARSE_STR	"coordinate"
+#define MM_COMPLEX_STR	"complex"
+#define MM_REAL_STR		"real"
+#define MM_INT_STR		"integer"
+#define MM_GENERAL_STR  "general"
+#define MM_SYMM_STR		"symmetric"
+#define MM_HERM_STR		"hermitian"
+#define MM_SKEW_STR		"skew-symmetric"
+#define MM_PATTERN_STR  "pattern"
+
+
+/*  high level routines */
+int mm_read_mtx_crd(char *fname, int *M, int *N, int *nz, int **I, int **J, 
+        double **val, MM_typecode *matcode);
+
+int mm_write_mtx_crd(char fname[], int M, int N, int nz, int I[], int J[],
+		 double val[], MM_typecode matcode);
+int mm_read_mtx_crd_data(FILE *f, int M, int N, int nz, int I[], int J[],
+		double val[], MM_typecode matcode);
+int mm_read_mtx_crd_entry(FILE *f, int *I, int *J, double *real, double *img,
+			MM_typecode matcode);
+
+int mm_read_unsymmetric_sparse(const char *fname, int *M_, int *N_, int *nz_,
+                double **val_, int **I_, int **J_);
+
+#if defined(__cplusplus)
+}
+#endif /* __cplusplus */  
+
+#endif

Samples/4_CUDA_Libraries/cuSolverSp_LowlevelCholesky/mmio_wrapper.cpp

@@ -0,0 +1,529 @@
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <math.h>
+
+#include "mmio.h"
+
+#include <cusolverDn.h>
+
+/* avoid Windows warnings (for example: strcpy, fscanf, etc.) */
+#if defined(_WIN32)  
+#define _CRT_SECURE_NO_WARNINGS
+#endif
+
+/* various __inline__ __device__  function to initialize a T_ELEM */
+template <typename T_ELEM> __inline__ T_ELEM cuGet (int );
+template <> __inline__ float cuGet<float >(int x)
+{
+    return float(x);
+}
+
+template <> __inline__ double cuGet<double>(int x)
+{
+    return double(x);
+}
+
+template <> __inline__ cuComplex cuGet<cuComplex>(int x)
+{
+    return (make_cuComplex( float(x), 0.0f ));
+}
+
+template <> __inline__ cuDoubleComplex  cuGet<cuDoubleComplex>(int x)
+{
+    return (make_cuDoubleComplex( double(x), 0.0 ));
+}
+
+
+template <typename T_ELEM> __inline__ T_ELEM cuGet (int , int );
+template <> __inline__ float cuGet<float >(int x, int y)
+{
+    return float(x);
+}
+
+template <> __inline__ double cuGet<double>(int x, int y)
+{
+    return double(x);
+}
+
+template <> __inline__ cuComplex cuGet<cuComplex>(int x, int y)
+{
+    return make_cuComplex( float(x), float(y) );
+}
+
+template <> __inline__ cuDoubleComplex  cuGet<cuDoubleComplex>(int x, int y)
+{
+    return (make_cuDoubleComplex( double(x), double(y) ));
+}
+
+
+template <typename T_ELEM> __inline__ T_ELEM cuGet (float );
+template <> __inline__ float cuGet<float >(float x)
+{
+    return float(x);
+}
+
+template <> __inline__ double cuGet<double>(float x)
+{
+    return double(x);
+}
+
+template <> __inline__ cuComplex cuGet<cuComplex>(float x)
+{
+    return (make_cuComplex( float(x), 0.0f ));
+}
+
+template <> __inline__ cuDoubleComplex  cuGet<cuDoubleComplex>(float x)
+{
+    return (make_cuDoubleComplex( double(x), 0.0 ));
+}
+
+
+template <typename T_ELEM> __inline__  T_ELEM cuGet (float, float );
+template <> __inline__  float cuGet<float >(float x, float y)
+{
+    return float(x);
+}
+
+template <> __inline__  double cuGet<double>(float x, float y)
+{
+    return double(x);
+}
+
+template <> __inline__  cuComplex cuGet<cuComplex>(float x, float y)
+{
+    return (make_cuComplex( float(x), float(y) ));
+}
+
+template <> __inline__  cuDoubleComplex  cuGet<cuDoubleComplex>(float x, float y)
+{
+    return (make_cuDoubleComplex( double(x), double(y) ));
+}
+
+
+template <typename T_ELEM> __inline__ T_ELEM cuGet (double );
+template <> __inline__ float cuGet<float >(double x)
+{
+    return float(x);
+}
+
+template <> __inline__ double cuGet<double>(double x)
+{
+    return double(x);
+}
+
+template <> __inline__ cuComplex cuGet<cuComplex>(double x)
+{
+    return (make_cuComplex( float(x), 0.0f ));
+}
+
+template <> __inline__ cuDoubleComplex  cuGet<cuDoubleComplex>(double x)
+{
+    return (make_cuDoubleComplex( double(x), 0.0 ));
+}
+
+
+template <typename T_ELEM> __inline__  T_ELEM cuGet (double, double );
+template <> __inline__  float cuGet<float >(double x, double y)
+{
+    return float(x);
+}
+
+template <> __inline__  double cuGet<double>(double x, double y)
+{
+    return double(x);
+}
+
+template <> __inline__  cuComplex cuGet<cuComplex>(double x, double y)
+{
+    return (make_cuComplex( float(x), float(y) ));
+}
+
+template <> __inline__  cuDoubleComplex  cuGet<cuDoubleComplex>(double x, double y)
+{
+    return (make_cuDoubleComplex( double(x), double(y) ));
+}
+
+
+
+
+
+static void compress_index(
+    const int *Ind, 
+    int nnz, 
+    int m, 
+    int *Ptr, 
+    int base)
+{
+    int i;
+
+    /* initialize everything to zero */
+    for(i=0; i<m+1; i++){
+        Ptr[i]=0;
+    } 
+    /* count elements in every row */
+    Ptr[0]=base;
+    for(i=0; i<nnz; i++){
+        Ptr[Ind[i]+(1-base)]++;
+    } 
+    /* add all the values */
+    for(i=0; i<m; i++){
+        Ptr[i+1]+=Ptr[i];
+    } 
+}
+
+
+struct cooFormat {
+    int i ;
+    int j ;
+    int p ; // permutation
+};
+
+
+int cmp_cooFormat_csr( struct cooFormat *s, struct cooFormat *t)
+{
+    if ( s->i < t->i ){
+        return -1 ;
+    }
+    else if ( s->i > t->i ){
+        return 1 ;
+    }
+    else{
+        return s->j - t->j ;
+    }
+}
+
+int cmp_cooFormat_csc( struct cooFormat *s, struct cooFormat *t)
+{
+    if ( s->j < t->j ){
+        return -1 ;
+    }
+    else if ( s->j > t->j ){
+        return 1 ;
+    }
+    else{
+        return s->i - t->i ;
+    }
+}
+
+typedef int (*FUNPTR) (const void*, const void*)  ;
+typedef int (*FUNPTR2) ( struct cooFormat *s, struct cooFormat *t)  ;
+
+static FUNPTR2  fptr_array[2] = {
+    cmp_cooFormat_csr,
+    cmp_cooFormat_csc,
+};
+
+
+static int verify_pattern(
+    int m,
+    int nnz,
+    int *csrRowPtr,
+    int *csrColInd)
+{
+    int i, col, start, end, base_index;
+    int error_found = 0;
+
+    if (nnz != (csrRowPtr[m] - csrRowPtr[0])){
+        fprintf(stderr, "Error (nnz check failed): (csrRowPtr[%d]=%d - csrRowPtr[%d]=%d) != (nnz=%d)\n", 0, csrRowPtr[0], m, csrRowPtr[m], nnz);
+        error_found = 1;
+    }
+
+    base_index = csrRowPtr[0];
+    if ((0 != base_index) && (1 != base_index)){
+        fprintf(stderr, "Error (base index check failed): base index = %d\n", base_index);
+        error_found = 1;
+    }
+
+    for (i=0; (!error_found) && (i<m); i++){
+        start = csrRowPtr[i  ] - base_index;
+        end   = csrRowPtr[i+1] - base_index;
+        if (start > end){
+            fprintf(stderr, "Error (corrupted row): csrRowPtr[%d] (=%d) > csrRowPtr[%d] (=%d)\n", i, start+base_index, i+1, end+base_index);
+            error_found = 1;
+        }
+        for (col=start; col<end; col++){
+            if (csrColInd[col] < base_index){
+                fprintf(stderr, "Error (column vs. base index check failed): csrColInd[%d] < %d\n", col, base_index);
+                error_found = 1;
+            }
+            if ((col < (end-1)) && (csrColInd[col] >= csrColInd[col+1])){
+                fprintf(stderr, "Error (sorting of the column indecis check failed): (csrColInd[%d]=%d) >= (csrColInd[%d]=%d)\n", col, csrColInd[col], col+1, csrColInd[col+1]);
+                error_found = 1;
+            }
+        }
+    }
+    return error_found ;
+}
+
+
+template <typename T_ELEM>
+int loadMMSparseMatrix(
+    char *filename, 
+    char elem_type, 
+    bool csrFormat, 
+    int *m, 
+    int *n, 
+    int *nnz, 
+    T_ELEM **aVal, 
+    int **aRowInd, 
+    int **aColInd, 
+    int extendSymMatrix)
+{
+    MM_typecode matcode;
+    double *tempVal;
+    int    *tempRowInd,*tempColInd;
+    double *tval;
+    int    *trow,*tcol;
+    int    *csrRowPtr, *cscColPtr;
+    int    i,j,error,base,count;
+    struct cooFormat *work;
+
+    /* read the matrix */   
+    error = mm_read_mtx_crd(filename, m, n, nnz, &trow, &tcol, &tval, &matcode);
+    if (error) {
+        fprintf(stderr, "!!!! can not open file: '%s'\n", filename);
+        return 1;       
+    }
+
+    /* start error checking */
+    if (mm_is_complex(matcode) && ((elem_type != 'z') && (elem_type != 'c'))) {
+        fprintf(stderr, "!!!! complex matrix requires type 'z' or 'c'\n");
+        return 1;            
+    }
+
+    if (mm_is_dense(matcode) || mm_is_array(matcode) || mm_is_pattern(matcode) /*|| mm_is_integer(matcode)*/){
+        fprintf(stderr, "!!!! dense, array, pattern and integer matrices are not supported\n");
+        return 1;     
+    }
+
+    /* if necessary symmetrize the pattern (transform from triangular to full) */
+    if ((extendSymMatrix) && (mm_is_symmetric(matcode) || mm_is_hermitian(matcode) || mm_is_skew(matcode))){
+        //count number of non-diagonal elements
+        count=0;
+        for(i=0; i<(*nnz); i++){
+            if (trow[i] != tcol[i]){
+                count++;
+            }
+        }
+        //allocate space for the symmetrized matrix
+        tempRowInd  =    (int *)malloc((*nnz + count) * sizeof(int));
+        tempColInd  =    (int *)malloc((*nnz + count) * sizeof(int));
+        if (mm_is_real(matcode) || mm_is_integer(matcode)){
+            tempVal = (double *)malloc((*nnz + count) * sizeof(double));
+        }
+        else{
+            tempVal = (double *)malloc(2 * (*nnz + count) * sizeof(double));
+        }
+        //copy the elements regular and transposed locations
+        for(j=0, i=0; i<(*nnz); i++){
+            tempRowInd[j]=trow[i]; 
+            tempColInd[j]=tcol[i];
+            if (mm_is_real(matcode) || mm_is_integer(matcode)){
+                tempVal[j]=tval[i];
+            }
+            else{
+                tempVal[2*j]  =tval[2*i];
+                tempVal[2*j+1]=tval[2*i+1];
+            }
+            j++;
+            if (trow[i] != tcol[i]){
+                tempRowInd[j]=tcol[i];
+                tempColInd[j]=trow[i];
+                if (mm_is_real(matcode) || mm_is_integer(matcode)){
+                    if (mm_is_skew(matcode)){
+                        tempVal[j]=-tval[i];
+                    }
+                    else{
+                        tempVal[j]= tval[i];
+                    }
+                }
+                else{
+                    if(mm_is_hermitian(matcode)){
+                        tempVal[2*j]  = tval[2*i];
+                        tempVal[2*j+1]=-tval[2*i+1];
+                    }
+                    else{
+                        tempVal[2*j]  = tval[2*i];
+                        tempVal[2*j+1]= tval[2*i+1];
+                    }
+                }
+                j++;
+            }
+        }
+        (*nnz)+=count;
+        //free temporary storage
+        free(trow);
+        free(tcol);
+        free(tval);        
+    }
+    else{
+        tempRowInd=trow;
+        tempColInd=tcol;
+        tempVal   =tval;
+    }
+    // life time of (trow, tcol, tval) is over.
+    // please use COO format (tempRowInd, tempColInd, tempVal)
+
+// use qsort to sort COO format 
+    work = (struct cooFormat *)malloc(sizeof(struct cooFormat)*(*nnz));
+    if (NULL == work){
+        fprintf(stderr, "!!!! allocation error, malloc failed\n");
+        return 1;
+    }
+    for(i=0; i<(*nnz); i++){
+        work[i].i = tempRowInd[i];
+        work[i].j = tempColInd[i];
+        work[i].p = i; // permutation is identity
+    }
+ 
+    if (csrFormat){
+        /* create row-major ordering of indices (sorted by row and within each row by column) */
+        qsort(work, *nnz, sizeof(struct cooFormat), (FUNPTR)fptr_array[0] );
+    }else{
+        /* create column-major ordering of indices (sorted by column and within each column by row) */
+        qsort(work, *nnz, sizeof(struct cooFormat), (FUNPTR)fptr_array[1] );
+
+    }
+
+    // (tempRowInd, tempColInd) is sorted either by row-major or by col-major
+    for(i=0; i<(*nnz); i++){
+        tempRowInd[i] = work[i].i;
+        tempColInd[i] = work[i].j;
+    }
+
+    // setup base 
+    // check if there is any row/col 0, if so base-0
+    // check if there is any row/col equal to matrix dimension m/n, if so base-1
+    int base0 = 0;
+    int base1 = 0;
+    for(i=0; i<(*nnz); i++){
+        const int row = tempRowInd[i];
+        const int col = tempColInd[i];
+        if ( (0 == row) || (0 == col) ){
+            base0 = 1;
+        }
+        if ( (*m == row) || (*n == col) ){
+            base1 = 1;
+        }
+    }
+    if ( base0 && base1 ){
+        printf("Error: input matrix is base-0 and base-1 \n");
+        return 1;
+    }
+
+    base = 0;
+    if (base1){
+        base = 1;
+    }
+
+    /* compress the appropriate indices */
+    if (csrFormat){
+        /* CSR format (assuming row-major format) */
+        csrRowPtr = (int *)malloc(((*m)+1) * sizeof(csrRowPtr[0]));
+        if (!csrRowPtr) return 1;          
+        compress_index(tempRowInd, *nnz, *m, csrRowPtr, base);
+
+        *aRowInd = csrRowPtr;
+        *aColInd = (int *)malloc((*nnz) * sizeof(int));
+    }
+    else {
+        /* CSC format (assuming column-major format) */
+        cscColPtr = (int *)malloc(((*n)+1) * sizeof(cscColPtr[0]));
+        if (!cscColPtr) return 1;          
+        compress_index(tempColInd, *nnz, *n, cscColPtr, base);
+
+        *aColInd = cscColPtr;
+        *aRowInd = (int *)malloc((*nnz) * sizeof(int));
+    }    
+
+    /* transfrom the matrix values of type double into one of the cusparse library types */ 
+    *aVal = (T_ELEM *)malloc((*nnz) * sizeof(T_ELEM));
+   
+    for (i=0; i<(*nnz); i++) {        
+        if (csrFormat){
+            (*aColInd)[i] = tempColInd[i];
+        }
+        else{
+            (*aRowInd)[i] = tempRowInd[i];
+        }
+        if (mm_is_real(matcode) || mm_is_integer(matcode)){
+            (*aVal)[i] = cuGet<T_ELEM>( tempVal[ work[i].p ] );
+        }
+        else{
+            (*aVal)[i] = cuGet<T_ELEM>(tempVal[2*work[i].p], tempVal[2*work[i].p+1]);
+        }
+    }
+
+    /* check for corruption */
+    int error_found;
+    if (csrFormat){
+        error_found = verify_pattern(*m, *nnz, *aRowInd, *aColInd);
+    }else{
+        error_found = verify_pattern(*n, *nnz, *aColInd, *aRowInd);
+    }
+    if (error_found){
+        fprintf(stderr, "!!!! verify_pattern failed\n");
+        return 1;
+    }
+
+    /* cleanup and exit */
+    free(work);
+    free(tempVal); 
+    free(tempColInd);
+    free(tempRowInd);
+
+    return 0;
+}   
+
+
+/* specific instantiation */
+template int loadMMSparseMatrix<float>(
+    char *filename, 
+    char elem_type, 
+    bool csrFormat, 
+    int *m, 
+    int *n, 
+    int *nnz, 
+    float  **aVal, 
+    int **aRowInd, 
+    int **aColInd, 
+    int extendSymMatrix);
+
+template int loadMMSparseMatrix<double>(
+    char *filename, 
+    char elem_type, 
+    bool csrFormat, 
+    int *m, 
+    int *n, 
+    int *nnz, 
+    double  **aVal, 
+    int **aRowInd, 
+    int **aColInd, 
+    int extendSymMatrix);
+
+template int loadMMSparseMatrix<cuComplex>(
+    char *filename, 
+    char elem_type, 
+    bool csrFormat, 
+    int *m, 
+    int *n, 
+    int *nnz, 
+    cuComplex  **aVal, 
+    int **aRowInd, 
+    int **aColInd, 
+    int extendSymMatrix);
+
+template int loadMMSparseMatrix<cuDoubleComplex>(
+    char *filename, 
+    char elem_type, 
+    bool csrFormat, 
+    int *m, 
+    int *n, 
+    int *nnz, 
+    cuDoubleComplex **aVal, 
+    int **aRowInd, 
+    int **aColInd, 
+    int extendSymMatrix);
+
+