CUDA 13.2 samples update (#432)

- Added Python samples for CUDA Python 1.0 release
- Renamed top-level `Samples` directory to `cpp` to accommodate Python samples.

cpp/2_Concepts_and_Techniques/eigenvalues/.vscode/c_cpp_properties.json

@@ -0,0 +1,18 @@
+{
+    "configurations": [
+        {
+            "name": "Linux",
+            "includePath": [
+                "${workspaceFolder}/**",
+                "${workspaceFolder}/../../../Common"
+            ],
+            "defines": [],
+            "compilerPath": "/usr/local/cuda/bin/nvcc",
+            "cStandard": "gnu17",
+            "cppStandard": "gnu++14",
+            "intelliSenseMode": "linux-gcc-x64",
+            "configurationProvider": "ms-vscode.makefile-tools"
+        }
+    ],
+    "version": 4
+}

cpp/2_Concepts_and_Techniques/eigenvalues/.vscode/extensions.json

@@ -0,0 +1,7 @@
+{
+    "recommendations": [
+        "nvidia.nsight-vscode-edition",
+        "ms-vscode.cpptools",
+        "ms-vscode.makefile-tools"
+    ]
+}

cpp/2_Concepts_and_Techniques/eigenvalues/CMakeLists.txt

@@ -0,0 +1,55 @@
+cmake_minimum_required(VERSION 3.20)
+
+list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/../../../cmake/Modules")
+
+project(eigenvalues LANGUAGES C CXX CUDA)
+
+find_package(CUDAToolkit REQUIRED)
+
+set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+
+set(CMAKE_CUDA_ARCHITECTURES 75 80 86 87 89 90 100 110 120)
+set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Wno-deprecated-gpu-targets")
+if(ENABLE_CUDA_DEBUG)
+    set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -G")        # enable cuda-gdb (may significantly affect performance on some targets)
+else()
+    set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -lineinfo") # add line information to all builds for debug tools (exclusive to -G option)
+endif()
+
+# Include directories and libraries
+include_directories(../../../Common)
+
+# Source file
+# Add target for eigenvalues
+add_executable(eigenvalues bisect_large.cu bisect_small.cu bisect_util.cu gerschgorin.cpp main.cu matlab.cpp)
+
+target_compile_options(eigenvalues PRIVATE $<$<COMPILE_LANGUAGE:CUDA>:--extended-lambda>)
+
+target_compile_features(eigenvalues PRIVATE cxx_std_17 cuda_std_17)
+
+set_target_properties(eigenvalues PROPERTIES CUDA_SEPARABLE_COMPILATION ON)
+
+# Copy data files to output directory
+add_custom_command(TARGET eigenvalues POST_BUILD
+    COMMAND ${CMAKE_COMMAND} -E copy_if_different
+    ${CMAKE_CURRENT_SOURCE_DIR}/data/diagonal.dat
+    ${CMAKE_CURRENT_BINARY_DIR}/
+)
+
+# Copy data files to output directory
+add_custom_command(TARGET eigenvalues POST_BUILD
+    COMMAND ${CMAKE_COMMAND} -E copy_if_different
+    ${CMAKE_CURRENT_SOURCE_DIR}/data/superdiagonal.dat
+    ${CMAKE_CURRENT_BINARY_DIR}/
+)
+
+# Copy data files to output directory
+add_custom_command(TARGET eigenvalues POST_BUILD
+    COMMAND ${CMAKE_COMMAND} -E copy_if_different
+    ${CMAKE_CURRENT_SOURCE_DIR}/data/reference.dat
+    ${CMAKE_CURRENT_BINARY_DIR}/
+)
+
+# Include installation configuration
+include(${CMAKE_CURRENT_SOURCE_DIR}/../../../cmake/InstallSamples.cmake)
+setup_samples_install()

cpp/2_Concepts_and_Techniques/eigenvalues/README.md

@@ -0,0 +1,34 @@
+# eigenvalues - Eigenvalues
+
+## Description
+
+The computation of all or a subset of all eigenvalues is an important problem in Linear Algebra, statistics, physics, and many other fields. This sample demonstrates a parallel implementation of a bisection algorithm for the computation of all eigenvalues of a tridiagonal symmetric matrix of arbitrary size with CUDA.
+
+## Key Concepts
+
+Linear Algebra
+
+## Supported SM Architectures
+
+[SM 5.0 ](https://developer.nvidia.com/cuda-gpus)  [SM 5.2 ](https://developer.nvidia.com/cuda-gpus)  [SM 5.3 ](https://developer.nvidia.com/cuda-gpus)  [SM 6.0 ](https://developer.nvidia.com/cuda-gpus)  [SM 6.1 ](https://developer.nvidia.com/cuda-gpus)  [SM 7.0 ](https://developer.nvidia.com/cuda-gpus)  [SM 7.2 ](https://developer.nvidia.com/cuda-gpus)  [SM 7.5 ](https://developer.nvidia.com/cuda-gpus)  [SM 8.0 ](https://developer.nvidia.com/cuda-gpus)  [SM 8.6 ](https://developer.nvidia.com/cuda-gpus)  [SM 8.7 ](https://developer.nvidia.com/cuda-gpus)  [SM 8.9 ](https://developer.nvidia.com/cuda-gpus)  [SM 9.0 ](https://developer.nvidia.com/cuda-gpus)
+
+## Supported OSes
+
+Linux, Windows
+
+## Supported CPU Architecture
+
+x86_64, armv7l
+
+## CUDA APIs involved
+
+### [CUDA Runtime API](http://docs.nvidia.com/cuda/cuda-runtime-api/index.html)
+cudaMalloc, cudaDeviceSynchronize, cudaMemcpy, cudaFree
+
+## Prerequisites
+
+Download and install the [CUDA Toolkit](https://developer.nvidia.com/cuda-downloads) for your corresponding platform.
+
+## References (for more details)
+
+[whitepaper](./doc/eigenvalues.pdf)

cpp/2_Concepts_and_Techniques/eigenvalues/bisect_kernel_large.cuh

@@ -0,0 +1,965 @@
+/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/* Determine eigenvalues for large symmetric, tridiagonal matrix. First
+  step of the computation. */
+
+#ifndef _BISECT_KERNEL_LARGE_H_
+#define _BISECT_KERNEL_LARGE_H_
+#include <cooperative_groups.h>
+
+namespace cg = cooperative_groups;
+// includes, project
+#include "config.h"
+#include "util.h"
+
+// additional kernel
+#include "bisect_util.cu"
+
+// declaration, forward
+
+////////////////////////////////////////////////////////////////////////////////
+//! Write data to global memory
+////////////////////////////////////////////////////////////////////////////////
+__device__ void writeToGmem(const unsigned int tid,
+                            const unsigned int tid_2,
+                            const unsigned int num_threads_active,
+                            const unsigned int num_blocks_mult,
+                            float             *g_left_one,
+                            float             *g_right_one,
+                            unsigned int      *g_pos_one,
+                            float             *g_left_mult,
+                            float             *g_right_mult,
+                            unsigned int      *g_left_count_mult,
+                            unsigned int      *g_right_count_mult,
+                            float             *s_left,
+                            float             *s_right,
+                            unsigned short    *s_left_count,
+                            unsigned short    *s_right_count,
+                            unsigned int      *g_blocks_mult,
+                            unsigned int      *g_blocks_mult_sum,
+                            unsigned short    *s_compaction_list,
+                            unsigned short    *s_cl_helper,
+                            unsigned int       offset_mult_lambda);
+
+////////////////////////////////////////////////////////////////////////////////
+//! Perform final stream compaction before writing out data
+////////////////////////////////////////////////////////////////////////////////
+__device__ void compactStreamsFinal(const unsigned int tid,
+                                    const unsigned int tid_2,
+                                    const unsigned int num_threads_active,
+                                    unsigned int      &offset_mult_lambda,
+                                    float             *s_left,
+                                    float             *s_right,
+                                    unsigned short    *s_left_count,
+                                    unsigned short    *s_right_count,
+                                    unsigned short    *s_cl_one,
+                                    unsigned short    *s_cl_mult,
+                                    unsigned short    *s_cl_blocking,
+                                    unsigned short    *s_cl_helper,
+                                    unsigned int       is_one_lambda,
+                                    unsigned int       is_one_lambda_2,
+                                    float             &left,
+                                    float             &right,
+                                    float             &left_2,
+                                    float             &right_2,
+                                    unsigned int      &left_count,
+                                    unsigned int      &right_count,
+                                    unsigned int      &left_count_2,
+                                    unsigned int      &right_count_2,
+                                    unsigned int       c_block_iend,
+                                    unsigned int       c_sum_block,
+                                    unsigned int       c_block_iend_2,
+                                    unsigned int       c_sum_block_2,
+                                    cg::thread_block   cta);
+
+////////////////////////////////////////////////////////////////////////////////
+//! Perform scan to compact list of block start addresses
+////////////////////////////////////////////////////////////////////////////////
+__device__ void scanCompactBlocksStartAddress(const unsigned int tid,
+                                              const unsigned int tid_2,
+                                              const unsigned int num_threads_compaction,
+                                              unsigned short    *s_cl_blocking,
+                                              unsigned short    *s_cl_helper,
+                                              cg::thread_block   cta);
+
+////////////////////////////////////////////////////////////////////////////////
+//! Perform scan to obtain number of eigenvalues before a specific block
+////////////////////////////////////////////////////////////////////////////////
+__device__ void scanSumBlocks(const unsigned int tid,
+                              const unsigned int tid_2,
+                              const unsigned int num_threads_active,
+                              const unsigned int num_threads_compaction,
+                              unsigned short    *s_cl_blocking,
+                              unsigned short    *s_cl_helper,
+                              cg::thread_block   cta);
+
+////////////////////////////////////////////////////////////////////////////////
+//! Perform initial scan for compaction of intervals containing one and
+//! multiple eigenvalues; also do initial scan to build blocks
+////////////////////////////////////////////////////////////////////////////////
+__device__ void scanInitial(const unsigned int tid,
+                            const unsigned int tid_2,
+                            const unsigned int num_threads_active,
+                            const unsigned int num_threads_compaction,
+                            unsigned short    *s_cl_one,
+                            unsigned short    *s_cl_mult,
+                            unsigned short    *s_cl_blocking,
+                            unsigned short    *s_cl_helper,
+                            cg::thread_block   cta);
+
+////////////////////////////////////////////////////////////////////////////////
+//! Store all non-empty intervals resulting from the subdivision of the interval
+//! currently processed by the thread
+//! @param  addr  address where to store
+////////////////////////////////////////////////////////////////////////////////
+__device__ void storeNonEmptyIntervalsLarge(unsigned int         addr,
+                                            const unsigned int   num_threads_active,
+                                            float               *s_left,
+                                            float               *s_right,
+                                            unsigned short      *s_left_count,
+                                            unsigned short      *s_right_count,
+                                            float                left,
+                                            float                mid,
+                                            float                right,
+                                            const unsigned short left_count,
+                                            const unsigned short mid_count,
+                                            const unsigned short right_count,
+                                            float                epsilon,
+                                            unsigned int        &compact_second_chunk,
+                                            unsigned short      *s_compaction_list,
+                                            unsigned int        &is_active_second);
+
+////////////////////////////////////////////////////////////////////////////////
+//! Bisection to find eigenvalues of a real, symmetric, and tridiagonal matrix
+//! @param  g_d  diagonal elements in global memory
+//! @param  g_s  superdiagonal elements in global elements (stored so that the
+//!              element *(g_s - 1) can be accessed an equals 0
+//! @param  n   size of matrix
+//! @param  lg  lower bound of input interval (e.g. Gerschgorin interval)
+//! @param  ug  upper bound of input interval (e.g. Gerschgorin interval)
+//! @param  lg_eig_count  number of eigenvalues that are smaller than \a lg
+//! @param  lu_eig_count  number of eigenvalues that are smaller than \a lu
+//! @param  epsilon  desired accuracy of eigenvalues to compute
+////////////////////////////////////////////////////////////////////////////////
+__global__ void bisectKernelLarge(float             *g_d,
+                                  float             *g_s,
+                                  const unsigned int n,
+                                  const float        lg,
+                                  const float        ug,
+                                  const unsigned int lg_eig_count,
+                                  const unsigned int ug_eig_count,
+                                  float              epsilon,
+                                  unsigned int      *g_num_one,
+                                  unsigned int      *g_num_blocks_mult,
+                                  float             *g_left_one,
+                                  float             *g_right_one,
+                                  unsigned int      *g_pos_one,
+                                  float             *g_left_mult,
+                                  float             *g_right_mult,
+                                  unsigned int      *g_left_count_mult,
+                                  unsigned int      *g_right_count_mult,
+                                  unsigned int      *g_blocks_mult,
+                                  unsigned int      *g_blocks_mult_sum)
+{
+    // Handle to thread block group
+    cg::thread_block   cta = cg::this_thread_block();
+    const unsigned int tid = threadIdx.x;
+
+    // intervals (store left and right because the subdivision tree is in general
+    // not dense
+    __shared__ float s_left[2 * MAX_THREADS_BLOCK + 1];
+    __shared__ float s_right[2 * MAX_THREADS_BLOCK + 1];
+
+    // number of eigenvalues that are smaller than s_left / s_right
+    // (correspondence is realized via indices)
+    __shared__ unsigned short s_left_count[2 * MAX_THREADS_BLOCK + 1];
+    __shared__ unsigned short s_right_count[2 * MAX_THREADS_BLOCK + 1];
+
+    // helper for stream compaction
+    __shared__ unsigned short s_compaction_list[2 * MAX_THREADS_BLOCK + 1];
+
+    // state variables for whole block
+    // if 0 then compaction of second chunk of child intervals is not necessary
+    // (because all intervals had exactly one non-dead child)
+    __shared__ unsigned int compact_second_chunk;
+    // if 1 then all threads are converged
+    __shared__ unsigned int all_threads_converged;
+
+    // number of currently active threads
+    __shared__ unsigned int num_threads_active;
+
+    // number of threads to use for stream compaction
+    __shared__ unsigned int num_threads_compaction;
+
+    // helper for exclusive scan
+    unsigned short *s_compaction_list_exc = s_compaction_list + 1;
+
+    // variables for currently processed interval
+    // left and right limit of active interval
+    float        left        = 0.0f;
+    float        right       = 0.0f;
+    unsigned int left_count  = 0;
+    unsigned int right_count = 0;
+    // midpoint of active interval
+    float mid = 0.0f;
+    // number of eigenvalues smaller then mid
+    unsigned int mid_count = 0;
+    // helper for stream compaction (tracking of threads generating second child)
+    unsigned int is_active_second = 0;
+
+    // initialize lists
+    s_compaction_list[tid] = 0;
+    s_left[tid]            = 0;
+    s_right[tid]           = 0;
+    s_left_count[tid]      = 0;
+    s_right_count[tid]     = 0;
+
+    cg::sync(cta);
+
+    // set up initial configuration
+    if (0 == tid) {
+        s_left[0]        = lg;
+        s_right[0]       = ug;
+        s_left_count[0]  = lg_eig_count;
+        s_right_count[0] = ug_eig_count;
+
+        compact_second_chunk = 0;
+        num_threads_active   = 1;
+
+        num_threads_compaction = 1;
+
+        all_threads_converged = 1;
+    }
+
+    cg::sync(cta);
+
+    // for all active threads read intervals from the last level
+    // the number of (worst case) active threads per level l is 2^l
+    while (true) {
+        subdivideActiveInterval(tid,
+                                s_left,
+                                s_right,
+                                s_left_count,
+                                s_right_count,
+                                num_threads_active,
+                                left,
+                                right,
+                                left_count,
+                                right_count,
+                                mid,
+                                all_threads_converged);
+
+        cg::sync(cta);
+
+        // check if done
+        if (1 == all_threads_converged) {
+            break;
+        }
+
+        // compute number of eigenvalues smaller than mid
+        // use all threads for reading the necessary matrix data from global
+        // memory
+        // use s_left and s_right as scratch space for diagonal and
+        // superdiagonal of matrix
+        mid_count = computeNumSmallerEigenvalsLarge(
+            g_d, g_s, n, mid, threadIdx.x, num_threads_active, s_left, s_right, (left == right), cta);
+
+        cg::sync(cta);
+
+        // store intervals
+        // for all threads store the first child interval in a continuous chunk of
+        // memory, and the second child interval -- if it exists -- in a second
+        // chunk; it is likely that all threads reach convergence up to
+        // \a epsilon at the same level; furthermore, for higher level most / all
+        // threads will have only one child, storing the first child compactly will
+        // (first) avoid to perform a compaction step on the first chunk, (second)
+        // make it for higher levels (when all threads / intervals have
+        // exactly one child)  unnecessary to perform a compaction of the second
+        // chunk
+        if (tid < num_threads_active) {
+            if (left != right) {
+                // store intervals
+                storeNonEmptyIntervalsLarge(tid,
+                                            num_threads_active,
+                                            s_left,
+                                            s_right,
+                                            s_left_count,
+                                            s_right_count,
+                                            left,
+                                            mid,
+                                            right,
+                                            left_count,
+                                            mid_count,
+                                            right_count,
+                                            epsilon,
+                                            compact_second_chunk,
+                                            s_compaction_list_exc,
+                                            is_active_second);
+            }
+            else {
+                // re-write converged interval (has to be stored again because s_left
+                // and s_right are used as scratch space for
+                // computeNumSmallerEigenvalsLarge()
+                s_left[tid]        = left;
+                s_right[tid]       = left;
+                s_left_count[tid]  = left_count;
+                s_right_count[tid] = right_count;
+
+                is_active_second = 0;
+            }
+        }
+
+        // necessary so that compact_second_chunk is up-to-date
+        cg::sync(cta);
+
+        // perform compaction of chunk where second children are stored
+        // scan of (num_threads_active / 2) elements, thus at most
+        // (num_threads_active / 4) threads are needed
+        if (compact_second_chunk > 0) {
+            // create indices for compaction
+            createIndicesCompaction(s_compaction_list_exc, num_threads_compaction, cta);
+
+            compactIntervals(s_left,
+                             s_right,
+                             s_left_count,
+                             s_right_count,
+                             mid,
+                             right,
+                             mid_count,
+                             right_count,
+                             s_compaction_list,
+                             num_threads_active,
+                             is_active_second);
+        }
+
+        cg::sync(cta);
+
+        // update state variables
+        if (0 == tid) {
+            // update number of active threads with result of reduction
+            num_threads_active += s_compaction_list[num_threads_active];
+            num_threads_compaction = ceilPow2(num_threads_active);
+
+            compact_second_chunk  = 0;
+            all_threads_converged = 1;
+        }
+
+        cg::sync(cta);
+
+        if (num_threads_compaction > blockDim.x) {
+            break;
+        }
+    }
+
+    cg::sync(cta);
+
+    // generate two lists of intervals; one with intervals that contain one
+    // eigenvalue (or are converged), and one with intervals that need further
+    // subdivision
+
+    // perform two scans in parallel
+
+    unsigned int left_count_2;
+    unsigned int right_count_2;
+
+    unsigned int tid_2 = tid + blockDim.x;
+
+    // cache in per thread registers so that s_left_count and s_right_count
+    // can be used for scans
+    left_count  = s_left_count[tid];
+    right_count = s_right_count[tid];
+
+    // some threads have to cache data for two intervals
+    if (tid_2 < num_threads_active) {
+        left_count_2  = s_left_count[tid_2];
+        right_count_2 = s_right_count[tid_2];
+    }
+
+    // compaction list for intervals containing one and multiple eigenvalues
+    // do not affect first element for exclusive scan
+    unsigned short *s_cl_one  = s_left_count + 1;
+    unsigned short *s_cl_mult = s_right_count + 1;
+
+    // compaction list for generating blocks of intervals containing multiple
+    // eigenvalues
+    unsigned short *s_cl_blocking = s_compaction_list_exc;
+    // helper compaction list for generating blocks of intervals
+    __shared__ unsigned short s_cl_helper[2 * MAX_THREADS_BLOCK + 1];
+
+    if (0 == tid) {
+        // set to 0 for exclusive scan
+        s_left_count[0]  = 0;
+        s_right_count[0] = 0;
+    }
+
+    cg::sync(cta);
+
+    // flag if interval contains one or multiple eigenvalues
+    unsigned int is_one_lambda   = 0;
+    unsigned int is_one_lambda_2 = 0;
+
+    // number of eigenvalues in the interval
+    unsigned int multiplicity = right_count - left_count;
+    is_one_lambda             = (1 == multiplicity);
+
+    s_cl_one[tid]  = is_one_lambda;
+    s_cl_mult[tid] = (!is_one_lambda);
+
+    // (note: s_cl_blocking is non-zero only where s_cl_mult[] is non-zero)
+    s_cl_blocking[tid] = (1 == is_one_lambda) ? 0 : multiplicity;
+    s_cl_helper[tid]   = 0;
+
+    if (tid_2 < num_threads_active) {
+        unsigned int multiplicity = right_count_2 - left_count_2;
+        is_one_lambda_2           = (1 == multiplicity);
+
+        s_cl_one[tid_2]  = is_one_lambda_2;
+        s_cl_mult[tid_2] = (!is_one_lambda_2);
+
+        // (note: s_cl_blocking is non-zero only where s_cl_mult[] is non-zero)
+        s_cl_blocking[tid_2] = (1 == is_one_lambda_2) ? 0 : multiplicity;
+        s_cl_helper[tid_2]   = 0;
+    }
+    else if (tid_2 < (2 * MAX_THREADS_BLOCK + 1)) {
+        // clear
+        s_cl_blocking[tid_2] = 0;
+        s_cl_helper[tid_2]   = 0;
+    }
+
+    scanInitial(
+        tid, tid_2, num_threads_active, num_threads_compaction, s_cl_one, s_cl_mult, s_cl_blocking, s_cl_helper, cta);
+
+    scanSumBlocks(tid, tid_2, num_threads_active, num_threads_compaction, s_cl_blocking, s_cl_helper, cta);
+
+    // end down sweep of scan
+    cg::sync(cta);
+
+    unsigned int c_block_iend   = 0;
+    unsigned int c_block_iend_2 = 0;
+    unsigned int c_sum_block    = 0;
+    unsigned int c_sum_block_2  = 0;
+
+    // for each thread / interval that corresponds to root node of interval block
+    // store start address of block and total number of eigenvalues in all blocks
+    // before this block (particular thread is irrelevant, constraint is to
+    // have a subset of threads so that one and only one of them is in each
+    // interval)
+    if (1 == s_cl_helper[tid]) {
+        c_block_iend = s_cl_mult[tid] + 1;
+        c_sum_block  = s_cl_blocking[tid];
+    }
+
+    if (1 == s_cl_helper[tid_2]) {
+        c_block_iend_2 = s_cl_mult[tid_2] + 1;
+        c_sum_block_2  = s_cl_blocking[tid_2];
+    }
+
+    scanCompactBlocksStartAddress(tid, tid_2, num_threads_compaction, s_cl_blocking, s_cl_helper, cta);
+
+    // finished second scan for s_cl_blocking
+    cg::sync(cta);
+
+    // determine the global results
+    __shared__ unsigned int num_blocks_mult;
+    __shared__ unsigned int num_mult;
+    __shared__ unsigned int offset_mult_lambda;
+
+    if (0 == tid) {
+        num_blocks_mult    = s_cl_blocking[num_threads_active - 1];
+        offset_mult_lambda = s_cl_one[num_threads_active - 1];
+        num_mult           = s_cl_mult[num_threads_active - 1];
+
+        *g_num_one         = offset_mult_lambda;
+        *g_num_blocks_mult = num_blocks_mult;
+    }
+
+    cg::sync(cta);
+
+    float left_2, right_2;
+    --s_cl_one;
+    --s_cl_mult;
+    --s_cl_blocking;
+    compactStreamsFinal(tid,
+                        tid_2,
+                        num_threads_active,
+                        offset_mult_lambda,
+                        s_left,
+                        s_right,
+                        s_left_count,
+                        s_right_count,
+                        s_cl_one,
+                        s_cl_mult,
+                        s_cl_blocking,
+                        s_cl_helper,
+                        is_one_lambda,
+                        is_one_lambda_2,
+                        left,
+                        right,
+                        left_2,
+                        right_2,
+                        left_count,
+                        right_count,
+                        left_count_2,
+                        right_count_2,
+                        c_block_iend,
+                        c_sum_block,
+                        c_block_iend_2,
+                        c_sum_block_2,
+                        cta);
+
+    cg::sync(cta);
+
+    // final adjustment before writing out data to global memory
+    if (0 == tid) {
+        s_cl_blocking[num_blocks_mult] = num_mult;
+        s_cl_helper[0]                 = 0;
+    }
+
+    cg::sync(cta);
+
+    // write to global memory
+    writeToGmem(tid,
+                tid_2,
+                num_threads_active,
+                num_blocks_mult,
+                g_left_one,
+                g_right_one,
+                g_pos_one,
+                g_left_mult,
+                g_right_mult,
+                g_left_count_mult,
+                g_right_count_mult,
+                s_left,
+                s_right,
+                s_left_count,
+                s_right_count,
+                g_blocks_mult,
+                g_blocks_mult_sum,
+                s_compaction_list,
+                s_cl_helper,
+                offset_mult_lambda);
+}
+
+////////////////////////////////////////////////////////////////////////////////
+//! Write data to global memory
+////////////////////////////////////////////////////////////////////////////////
+__device__ void writeToGmem(const unsigned int tid,
+                            const unsigned int tid_2,
+                            const unsigned int num_threads_active,
+                            const unsigned int num_blocks_mult,
+                            float             *g_left_one,
+                            float             *g_right_one,
+                            unsigned int      *g_pos_one,
+                            float             *g_left_mult,
+                            float             *g_right_mult,
+                            unsigned int      *g_left_count_mult,
+                            unsigned int      *g_right_count_mult,
+                            float             *s_left,
+                            float             *s_right,
+                            unsigned short    *s_left_count,
+                            unsigned short    *s_right_count,
+                            unsigned int      *g_blocks_mult,
+                            unsigned int      *g_blocks_mult_sum,
+                            unsigned short    *s_compaction_list,
+                            unsigned short    *s_cl_helper,
+                            unsigned int       offset_mult_lambda)
+{
+    if (tid < offset_mult_lambda) {
+        g_left_one[tid]  = s_left[tid];
+        g_right_one[tid] = s_right[tid];
+        // right count can be used to order eigenvalues without sorting
+        g_pos_one[tid] = s_right_count[tid];
+    }
+    else {
+        g_left_mult[tid - offset_mult_lambda]        = s_left[tid];
+        g_right_mult[tid - offset_mult_lambda]       = s_right[tid];
+        g_left_count_mult[tid - offset_mult_lambda]  = s_left_count[tid];
+        g_right_count_mult[tid - offset_mult_lambda] = s_right_count[tid];
+    }
+
+    if (tid_2 < num_threads_active) {
+        if (tid_2 < offset_mult_lambda) {
+            g_left_one[tid_2]  = s_left[tid_2];
+            g_right_one[tid_2] = s_right[tid_2];
+            // right count can be used to order eigenvalues without sorting
+            g_pos_one[tid_2] = s_right_count[tid_2];
+        }
+        else {
+            g_left_mult[tid_2 - offset_mult_lambda]        = s_left[tid_2];
+            g_right_mult[tid_2 - offset_mult_lambda]       = s_right[tid_2];
+            g_left_count_mult[tid_2 - offset_mult_lambda]  = s_left_count[tid_2];
+            g_right_count_mult[tid_2 - offset_mult_lambda] = s_right_count[tid_2];
+        }
+
+    } // end writing out data
+
+    // note that s_cl_blocking = s_compaction_list + 1;, that is by writing out
+    // s_compaction_list we write the exclusive scan result
+    if (tid <= num_blocks_mult) {
+        g_blocks_mult[tid]     = s_compaction_list[tid];
+        g_blocks_mult_sum[tid] = s_cl_helper[tid];
+    }
+
+    if (tid_2 <= num_blocks_mult) {
+        g_blocks_mult[tid_2]     = s_compaction_list[tid_2];
+        g_blocks_mult_sum[tid_2] = s_cl_helper[tid_2];
+    }
+}
+
+////////////////////////////////////////////////////////////////////////////////
+//! Perform final stream compaction before writing data to global memory
+////////////////////////////////////////////////////////////////////////////////
+__device__ void compactStreamsFinal(const unsigned int tid,
+                                    const unsigned int tid_2,
+                                    const unsigned int num_threads_active,
+                                    unsigned int      &offset_mult_lambda,
+                                    float             *s_left,
+                                    float             *s_right,
+                                    unsigned short    *s_left_count,
+                                    unsigned short    *s_right_count,
+                                    unsigned short    *s_cl_one,
+                                    unsigned short    *s_cl_mult,
+                                    unsigned short    *s_cl_blocking,
+                                    unsigned short    *s_cl_helper,
+                                    unsigned int       is_one_lambda,
+                                    unsigned int       is_one_lambda_2,
+                                    float             &left,
+                                    float             &right,
+                                    float             &left_2,
+                                    float             &right_2,
+                                    unsigned int      &left_count,
+                                    unsigned int      &right_count,
+                                    unsigned int      &left_count_2,
+                                    unsigned int      &right_count_2,
+                                    unsigned int       c_block_iend,
+                                    unsigned int       c_sum_block,
+                                    unsigned int       c_block_iend_2,
+                                    unsigned int       c_sum_block_2,
+                                    cg::thread_block   cta)
+{
+    // cache data before performing compaction
+    left  = s_left[tid];
+    right = s_right[tid];
+
+    if (tid_2 < num_threads_active) {
+        left_2  = s_left[tid_2];
+        right_2 = s_right[tid_2];
+    }
+
+    cg::sync(cta);
+
+    // determine addresses for intervals containing multiple eigenvalues and
+    // addresses for blocks of intervals
+    unsigned int ptr_w            = 0;
+    unsigned int ptr_w_2          = 0;
+    unsigned int ptr_blocking_w   = 0;
+    unsigned int ptr_blocking_w_2 = 0;
+
+    ptr_w = (1 == is_one_lambda) ? s_cl_one[tid] : s_cl_mult[tid] + offset_mult_lambda;
+
+    if (0 != c_block_iend) {
+        ptr_blocking_w = s_cl_blocking[tid];
+    }
+
+    if (tid_2 < num_threads_active) {
+        ptr_w_2 = (1 == is_one_lambda_2) ? s_cl_one[tid_2] : s_cl_mult[tid_2] + offset_mult_lambda;
+
+        if (0 != c_block_iend_2) {
+            ptr_blocking_w_2 = s_cl_blocking[tid_2];
+        }
+    }
+
+    cg::sync(cta);
+
+    // store compactly in shared mem
+    s_left[ptr_w]        = left;
+    s_right[ptr_w]       = right;
+    s_left_count[ptr_w]  = left_count;
+    s_right_count[ptr_w] = right_count;
+
+    if (0 != c_block_iend) {
+        s_cl_blocking[ptr_blocking_w + 1] = c_block_iend - 1;
+        s_cl_helper[ptr_blocking_w + 1]   = c_sum_block;
+    }
+
+    if (tid_2 < num_threads_active) {
+        // store compactly in shared mem
+        s_left[ptr_w_2]        = left_2;
+        s_right[ptr_w_2]       = right_2;
+        s_left_count[ptr_w_2]  = left_count_2;
+        s_right_count[ptr_w_2] = right_count_2;
+
+        if (0 != c_block_iend_2) {
+            s_cl_blocking[ptr_blocking_w_2 + 1] = c_block_iend_2 - 1;
+            s_cl_helper[ptr_blocking_w_2 + 1]   = c_sum_block_2;
+        }
+    }
+}
+
+////////////////////////////////////////////////////////////////////////////////
+//! Compute addresses to obtain compact list of block start addresses
+////////////////////////////////////////////////////////////////////////////////
+__device__ void scanCompactBlocksStartAddress(const unsigned int tid,
+                                              const unsigned int tid_2,
+                                              const unsigned int num_threads_compaction,
+                                              unsigned short    *s_cl_blocking,
+                                              unsigned short    *s_cl_helper,
+                                              cg::thread_block   cta)
+{
+    // prepare for second step of block generation: compaction of the block
+    // list itself to efficiently write out these
+    s_cl_blocking[tid] = s_cl_helper[tid];
+
+    if (tid_2 < num_threads_compaction) {
+        s_cl_blocking[tid_2] = s_cl_helper[tid_2];
+    }
+
+    cg::sync(cta);
+
+    // additional scan to compact s_cl_blocking that permits to generate a
+    // compact list of eigenvalue blocks each one containing about
+    // MAX_THREADS_BLOCK eigenvalues (so that each of these blocks may be
+    // processed by one thread block in a subsequent processing step
+
+    unsigned int offset = 1;
+
+    // build scan tree
+    for (int d = (num_threads_compaction >> 1); d > 0; d >>= 1) {
+        cg::sync(cta);
+
+        if (tid < d) {
+            unsigned int ai   = offset * (2 * tid + 1) - 1;
+            unsigned int bi   = offset * (2 * tid + 2) - 1;
+            s_cl_blocking[bi] = s_cl_blocking[bi] + s_cl_blocking[ai];
+        }
+
+        offset <<= 1;
+    }
+
+    // traverse down tree: first down to level 2 across
+    for (int d = 2; d < num_threads_compaction; d <<= 1) {
+        offset >>= 1;
+        cg::sync(cta);
+
+        //
+        if (tid < (d - 1)) {
+            unsigned int ai   = offset * (tid + 1) - 1;
+            unsigned int bi   = ai + (offset >> 1);
+            s_cl_blocking[bi] = s_cl_blocking[bi] + s_cl_blocking[ai];
+        }
+    }
+}
+
+////////////////////////////////////////////////////////////////////////////////
+//! Perform scan to obtain number of eigenvalues before a specific block
+////////////////////////////////////////////////////////////////////////////////
+__device__ void scanSumBlocks(const unsigned int tid,
+                              const unsigned int tid_2,
+                              const unsigned int num_threads_active,
+                              const unsigned int num_threads_compaction,
+                              unsigned short    *s_cl_blocking,
+                              unsigned short    *s_cl_helper,
+                              cg::thread_block   cta)
+{
+    unsigned int offset = 1;
+
+    // first step of scan to build the sum of elements within each block
+    // build up tree
+    for (int d = num_threads_compaction >> 1; d > 0; d >>= 1) {
+        cg::sync(cta);
+
+        if (tid < d) {
+            unsigned int ai = offset * (2 * tid + 1) - 1;
+            unsigned int bi = offset * (2 * tid + 2) - 1;
+
+            s_cl_blocking[bi] += s_cl_blocking[ai];
+        }
+
+        offset *= 2;
+    }
+
+    // first step of scan to build the sum of elements within each block
+    // traverse down tree
+    for (int d = 2; d < (num_threads_compaction - 1); d <<= 1) {
+        offset >>= 1;
+        cg::sync(cta);
+
+        if (tid < (d - 1)) {
+            unsigned int ai = offset * (tid + 1) - 1;
+            unsigned int bi = ai + (offset >> 1);
+
+            s_cl_blocking[bi] += s_cl_blocking[ai];
+        }
+    }
+
+    cg::sync(cta);
+
+    if (0 == tid) {
+        // move last element of scan to last element that is valid
+        // necessary because the number of threads employed for scan is a power
+        // of two and not necessarily the number of active threasd
+        s_cl_helper[num_threads_active - 1]   = s_cl_helper[num_threads_compaction - 1];
+        s_cl_blocking[num_threads_active - 1] = s_cl_blocking[num_threads_compaction - 1];
+    }
+}
+
+////////////////////////////////////////////////////////////////////////////////
+//! Perform initial scan for compaction of intervals containing one and
+//! multiple eigenvalues; also do initial scan to build blocks
+////////////////////////////////////////////////////////////////////////////////
+__device__ void scanInitial(const unsigned int tid,
+                            const unsigned int tid_2,
+                            const unsigned int num_threads_active,
+                            const unsigned int num_threads_compaction,
+                            unsigned short    *s_cl_one,
+                            unsigned short    *s_cl_mult,
+                            unsigned short    *s_cl_blocking,
+                            unsigned short    *s_cl_helper,
+                            cg::thread_block   cta)
+{
+    // perform scan to compactly write out the intervals containing one and
+    // multiple eigenvalues
+    // also generate tree for blocking of intervals containing multiple
+    // eigenvalues
+
+    unsigned int offset = 1;
+
+    // build scan tree
+    for (int d = (num_threads_compaction >> 1); d > 0; d >>= 1) {
+        cg::sync(cta);
+
+        if (tid < d) {
+            unsigned int ai = offset * (2 * tid + 1);
+            unsigned int bi = offset * (2 * tid + 2) - 1;
+
+            s_cl_one[bi]  = s_cl_one[bi] + s_cl_one[ai - 1];
+            s_cl_mult[bi] = s_cl_mult[bi] + s_cl_mult[ai - 1];
+
+            // s_cl_helper is binary and zero for an internal node and 1 for a
+            // root node of a tree corresponding to a block
+            // s_cl_blocking contains the number of nodes in each sub-tree at each
+            // iteration, the data has to be kept to compute the total number of
+            // eigenvalues per block that, in turn, is needed to efficiently
+            // write out data in the second step
+            if ((s_cl_helper[ai - 1] != 1) || (s_cl_helper[bi] != 1)) {
+                // check how many childs are non terminated
+                if (s_cl_helper[ai - 1] == 1) {
+                    // mark as terminated
+                    s_cl_helper[bi] = 1;
+                }
+                else if (s_cl_helper[bi] == 1) {
+                    // mark as terminated
+                    s_cl_helper[ai - 1] = 1;
+                }
+                else // both childs are non-terminated
+                {
+                    unsigned int temp = s_cl_blocking[bi] + s_cl_blocking[ai - 1];
+
+                    if (temp > MAX_THREADS_BLOCK) {
+                        // the two child trees have to form separate blocks, terminate trees
+                        s_cl_helper[ai - 1] = 1;
+                        s_cl_helper[bi]     = 1;
+                    }
+                    else {
+                        // build up tree by joining subtrees
+                        s_cl_blocking[bi]     = temp;
+                        s_cl_blocking[ai - 1] = 0;
+                    }
+                }
+            } // end s_cl_helper update
+        }
+
+        offset <<= 1;
+    }
+
+    // traverse down tree, this only for stream compaction, not for block
+    // construction
+    for (int d = 2; d < num_threads_compaction; d <<= 1) {
+        offset >>= 1;
+        cg::sync(cta);
+
+        //
+        if (tid < (d - 1)) {
+            unsigned int ai = offset * (tid + 1) - 1;
+            unsigned int bi = ai + (offset >> 1);
+
+            s_cl_one[bi]  = s_cl_one[bi] + s_cl_one[ai];
+            s_cl_mult[bi] = s_cl_mult[bi] + s_cl_mult[ai];
+        }
+    }
+}
+
+////////////////////////////////////////////////////////////////////////////////
+//! Store all non-empty intervals resulting from the subdivision of the interval
+//! currently processed by the thread
+////////////////////////////////////////////////////////////////////////////////
+__device__ void storeNonEmptyIntervalsLarge(unsigned int         addr,
+                                            const unsigned int   num_threads_active,
+                                            float               *s_left,
+                                            float               *s_right,
+                                            unsigned short      *s_left_count,
+                                            unsigned short      *s_right_count,
+                                            float                left,
+                                            float                mid,
+                                            float                right,
+                                            const unsigned short left_count,
+                                            const unsigned short mid_count,
+                                            const unsigned short right_count,
+                                            float                epsilon,
+                                            unsigned int        &compact_second_chunk,
+                                            unsigned short      *s_compaction_list,
+                                            unsigned int        &is_active_second)
+{
+    // check if both child intervals are valid
+    if ((left_count != mid_count) && (mid_count != right_count)) {
+        storeInterval(addr, s_left, s_right, s_left_count, s_right_count, left, mid, left_count, mid_count, epsilon);
+
+        is_active_second               = 1;
+        s_compaction_list[threadIdx.x] = 1;
+        atomicExch(&compact_second_chunk, 1);
+    }
+    else {
+        // only one non-empty child interval
+
+        // mark that no second child
+        is_active_second               = 0;
+        s_compaction_list[threadIdx.x] = 0;
+
+        // store the one valid child interval
+        if (left_count != mid_count) {
+            storeInterval(
+                addr, s_left, s_right, s_left_count, s_right_count, left, mid, left_count, mid_count, epsilon);
+        }
+        else {
+            storeInterval(
+                addr, s_left, s_right, s_left_count, s_right_count, mid, right, mid_count, right_count, epsilon);
+        }
+    }
+}
+
+#endif // #ifndef _BISECT_KERNEL_LARGE_H_

cpp/2_Concepts_and_Techniques/eigenvalues/bisect_kernel_large_multi.cuh

@@ -0,0 +1,268 @@
+/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/* Perform second step of bisection algorithm for large matrices for
+ * intervals that contained after the first step more than one eigenvalue
+ */
+
+#ifndef _BISECT_KERNEL_LARGE_MULTI_H_
+#define _BISECT_KERNEL_LARGE_MULTI_H_
+
+#include <cooperative_groups.h>
+
+namespace cg = cooperative_groups;
+// includes, project
+#include "config.h"
+#include "util.h"
+
+// additional kernel
+#include "bisect_util.cu"
+
+////////////////////////////////////////////////////////////////////////////////
+//! Perform second step of bisection algorithm for large matrices for
+//! intervals that after the first step contained more than one eigenvalue
+//! @param  g_d  diagonal elements of symmetric, tridiagonal matrix
+//! @param  g_s  superdiagonal elements of symmetric, tridiagonal matrix
+//! @param  n    matrix size
+//! @param  blocks_mult  start addresses of blocks of intervals that are
+//!                      processed by one block of threads, each of the
+//!                      intervals contains more than one eigenvalue
+//! @param  blocks_mult_sum  total number of eigenvalues / singleton intervals
+//!                          in one block of intervals
+//! @param  g_left  left limits of intervals
+//! @param  g_right  right limits of intervals
+//! @param  g_left_count  number of eigenvalues less than left limits
+//! @param  g_right_count  number of eigenvalues less than right limits
+//! @param  g_lambda  final eigenvalue
+//! @param  g_pos  index of eigenvalue (in ascending order)
+//! @param  precision  desired precision of eigenvalues
+////////////////////////////////////////////////////////////////////////////////
+__global__ void bisectKernelLarge_MultIntervals(float             *g_d,
+                                                float             *g_s,
+                                                const unsigned int n,
+                                                unsigned int      *blocks_mult,
+                                                unsigned int      *blocks_mult_sum,
+                                                float             *g_left,
+                                                float             *g_right,
+                                                unsigned int      *g_left_count,
+                                                unsigned int      *g_right_count,
+                                                float             *g_lambda,
+                                                unsigned int      *g_pos,
+                                                float              precision)
+{
+    // Handle to thread block group
+    cg::thread_block   cta = cg::this_thread_block();
+    const unsigned int tid = threadIdx.x;
+
+    // left and right limits of interval
+    __shared__ float s_left[2 * MAX_THREADS_BLOCK];
+    __shared__ float s_right[2 * MAX_THREADS_BLOCK];
+
+    // number of eigenvalues smaller than interval limits
+    __shared__ unsigned int s_left_count[2 * MAX_THREADS_BLOCK];
+    __shared__ unsigned int s_right_count[2 * MAX_THREADS_BLOCK];
+
+    // helper array for chunk compaction of second chunk
+    __shared__ unsigned int s_compaction_list[2 * MAX_THREADS_BLOCK + 1];
+    // compaction list helper for exclusive scan
+    unsigned int *s_compaction_list_exc = s_compaction_list + 1;
+
+    // flag if all threads are converged
+    __shared__ unsigned int all_threads_converged;
+    // number of active threads
+    __shared__ unsigned int num_threads_active;
+    // number of threads to employ for compaction
+    __shared__ unsigned int num_threads_compaction;
+    // flag if second chunk has to be compacted
+    __shared__ unsigned int compact_second_chunk;
+
+    // parameters of block of intervals processed by this block of threads
+    __shared__ unsigned int c_block_start;
+    __shared__ unsigned int c_block_end;
+    __shared__ unsigned int c_block_offset_output;
+
+    // midpoint of currently active interval of the thread
+    float mid = 0.0f;
+    // number of eigenvalues smaller than \a mid
+    unsigned int mid_count = 0;
+    // current interval parameter
+    float        left;
+    float        right;
+    unsigned int left_count;
+    unsigned int right_count;
+    // helper for compaction, keep track which threads have a second child
+    unsigned int is_active_second = 0;
+
+    // initialize common start conditions
+    if (0 == tid) {
+        c_block_start         = blocks_mult[blockIdx.x];
+        c_block_end           = blocks_mult[blockIdx.x + 1];
+        c_block_offset_output = blocks_mult_sum[blockIdx.x];
+
+        num_threads_active     = c_block_end - c_block_start;
+        s_compaction_list[0]   = 0;
+        num_threads_compaction = ceilPow2(num_threads_active);
+
+        all_threads_converged = 1;
+        compact_second_chunk  = 0;
+    }
+
+    cg::sync(cta);
+
+    // read data into shared memory
+    if (tid < num_threads_active) {
+        s_left[tid]        = g_left[c_block_start + tid];
+        s_right[tid]       = g_right[c_block_start + tid];
+        s_left_count[tid]  = g_left_count[c_block_start + tid];
+        s_right_count[tid] = g_right_count[c_block_start + tid];
+    }
+
+    cg::sync(cta);
+
+    // do until all threads converged
+    while (true) {
+        // for (int iter=0; iter < 0; iter++) {
+
+        // subdivide interval if currently active and not already converged
+        subdivideActiveInterval(tid,
+                                s_left,
+                                s_right,
+                                s_left_count,
+                                s_right_count,
+                                num_threads_active,
+                                left,
+                                right,
+                                left_count,
+                                right_count,
+                                mid,
+                                all_threads_converged);
+
+        cg::sync(cta);
+
+        // stop if all eigenvalues have been found
+        if (1 == all_threads_converged) {
+            break;
+        }
+
+        // compute number of eigenvalues smaller than mid for active and not
+        // converged intervals, use all threads for loading data from gmem and
+        // s_left and s_right as scratch space to store the data load from gmem
+        // in shared memory
+        mid_count = computeNumSmallerEigenvalsLarge(
+            g_d, g_s, n, mid, tid, num_threads_active, s_left, s_right, (left == right), cta);
+
+        cg::sync(cta);
+
+        if (tid < num_threads_active) {
+            // store intervals
+            if (left != right) {
+                storeNonEmptyIntervals(tid,
+                                       num_threads_active,
+                                       s_left,
+                                       s_right,
+                                       s_left_count,
+                                       s_right_count,
+                                       left,
+                                       mid,
+                                       right,
+                                       left_count,
+                                       mid_count,
+                                       right_count,
+                                       precision,
+                                       compact_second_chunk,
+                                       s_compaction_list_exc,
+                                       is_active_second);
+            }
+            else {
+                storeIntervalConverged(s_left,
+                                       s_right,
+                                       s_left_count,
+                                       s_right_count,
+                                       left,
+                                       mid,
+                                       right,
+                                       left_count,
+                                       mid_count,
+                                       right_count,
+                                       s_compaction_list_exc,
+                                       compact_second_chunk,
+                                       num_threads_active,
+                                       is_active_second);
+            }
+        }
+
+        cg::sync(cta);
+
+        // compact second chunk of intervals if any of the threads generated
+        // two child intervals
+        if (1 == compact_second_chunk) {
+            createIndicesCompaction(s_compaction_list_exc, num_threads_compaction, cta);
+
+            compactIntervals(s_left,
+                             s_right,
+                             s_left_count,
+                             s_right_count,
+                             mid,
+                             right,
+                             mid_count,
+                             right_count,
+                             s_compaction_list,
+                             num_threads_active,
+                             is_active_second);
+        }
+
+        cg::sync(cta);
+
+        // update state variables
+        if (0 == tid) {
+            num_threads_active += s_compaction_list[num_threads_active];
+            num_threads_compaction = ceilPow2(num_threads_active);
+
+            compact_second_chunk  = 0;
+            all_threads_converged = 1;
+        }
+
+        cg::sync(cta);
+
+        // clear
+        s_compaction_list_exc[threadIdx.x]              = 0;
+        s_compaction_list_exc[threadIdx.x + blockDim.x] = 0;
+
+        cg::sync(cta);
+
+    } // end until all threads converged
+
+    // write data back to global memory
+    if (tid < num_threads_active) {
+        unsigned int addr = c_block_offset_output + tid;
+
+        g_lambda[addr] = s_left[tid];
+        g_pos[addr]    = s_right_count[tid];
+    }
+}
+
+#endif // #ifndef _BISECT_KERNEL_LARGE_MULTI_H_

cpp/2_Concepts_and_Techniques/eigenvalues/bisect_kernel_large_onei.cuh

@@ -0,0 +1,167 @@
+/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/* Determine eigenvalues for large matrices for intervals that contained after
+ * the first step one eigenvalue
+ */
+
+#ifndef _BISECT_KERNEL_LARGE_ONEI_H_
+#define _BISECT_KERNEL_LARGE_ONEI_H_
+
+#include <cooperative_groups.h>
+
+namespace cg = cooperative_groups;
+
+// includes, project
+#include "config.h"
+#include "util.h"
+
+// additional kernel
+#include "bisect_util.cu"
+
+////////////////////////////////////////////////////////////////////////////////
+//! Determine eigenvalues for large matrices for intervals that after
+//! the first step contained one eigenvalue
+//! @param  g_d  diagonal elements of symmetric, tridiagonal matrix
+//! @param  g_s  superdiagonal elements of symmetric, tridiagonal matrix
+//! @param  n    matrix size
+//! @param  num_intervals  total number of intervals containing one eigenvalue
+//!                         after the first step
+//! @param g_left  left interval limits
+//! @param g_right  right interval limits
+//! @param g_pos  index of interval / number of intervals that are smaller than
+//!               right interval limit
+//! @param  precision  desired precision of eigenvalues
+////////////////////////////////////////////////////////////////////////////////
+__global__ void bisectKernelLarge_OneIntervals(float             *g_d,
+                                               float             *g_s,
+                                               const unsigned int n,
+                                               unsigned int       num_intervals,
+                                               float             *g_left,
+                                               float             *g_right,
+                                               unsigned int      *g_pos,
+                                               float              precision)
+{
+    // Handle to thread block group
+    cg::thread_block   cta  = cg::this_thread_block();
+    const unsigned int gtid = (blockDim.x * blockIdx.x) + threadIdx.x;
+
+    __shared__ float s_left_scratch[MAX_THREADS_BLOCK];
+    __shared__ float s_right_scratch[MAX_THREADS_BLOCK];
+
+    // active interval of thread
+    // left and right limit of current interval
+    float left, right;
+    // number of threads smaller than the right limit (also corresponds to the
+    // global index of the eigenvalues contained in the active interval)
+    unsigned int right_count;
+    // flag if current thread converged
+    unsigned int converged = 0;
+    // midpoint when current interval is subdivided
+    float mid = 0.0f;
+    // number of eigenvalues less than mid
+    unsigned int mid_count = 0;
+
+    // read data from global memory
+    if (gtid < num_intervals) {
+        left        = g_left[gtid];
+        right       = g_right[gtid];
+        right_count = g_pos[gtid];
+    }
+
+    // flag to determine if all threads converged to eigenvalue
+    __shared__ unsigned int converged_all_threads;
+
+    // initialized shared flag
+    if (0 == threadIdx.x) {
+        converged_all_threads = 0;
+    }
+
+    cg::sync(cta);
+
+    // process until all threads converged to an eigenvalue
+    // while( 0 == converged_all_threads) {
+    while (true) {
+        atomicExch(&converged_all_threads, 1);
+
+        // update midpoint for all active threads
+        if ((gtid < num_intervals) && (0 == converged)) {
+            mid = computeMidpoint(left, right);
+        }
+
+        // find number of eigenvalues that are smaller than midpoint
+        mid_count = computeNumSmallerEigenvalsLarge(
+            g_d, g_s, n, mid, gtid, num_intervals, s_left_scratch, s_right_scratch, converged, cta);
+
+        cg::sync(cta);
+
+        // for all active threads
+        if ((gtid < num_intervals) && (0 == converged)) {
+            // udpate intervals -- always one child interval survives
+            if (right_count == mid_count) {
+                right = mid;
+            }
+            else {
+                left = mid;
+            }
+
+            // check for convergence
+            float t0 = right - left;
+            float t1 = max(abs(right), abs(left)) * precision;
+
+            if (t0 < min(precision, t1)) {
+                float lambda = computeMidpoint(left, right);
+                left         = lambda;
+                right        = lambda;
+
+                converged = 1;
+            }
+            else {
+                atomicExch(&converged_all_threads, 0);
+            }
+        }
+
+        cg::sync(cta);
+
+        if (1 == converged_all_threads) {
+            break;
+        }
+
+        cg::sync(cta);
+    }
+
+    // write data back to global memory
+    cg::sync(cta);
+
+    if (gtid < num_intervals) {
+        // intervals converged so left and right interval limit are both identical
+        // and identical to the eigenvalue
+        g_left[gtid] = left;
+    }
+}
+
+#endif // #ifndef _BISECT_KERNEL_LARGE_ONEI_H_

cpp/2_Concepts_and_Techniques/eigenvalues/bisect_kernel_small.cuh

@@ -0,0 +1,271 @@
+/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/* Determine eigenvalues for small symmetric, tridiagonal matrix */
+
+#ifndef _BISECT_KERNEL_SMALL_H_
+#define _BISECT_KERNEL_SMALL_H_
+
+#include <cooperative_groups.h>
+
+namespace cg = cooperative_groups;
+
+// includes, project
+#include "config.h"
+#include "util.h"
+
+// additional kernel
+#include "bisect_util.cu"
+
+////////////////////////////////////////////////////////////////////////////////
+//! Bisection to find eigenvalues of a real, symmetric, and tridiagonal matrix
+//! @param  g_d  diagonal elements in global memory
+//! @param  g_s  superdiagonal elements in global elements (stored so that the
+//!              element *(g_s - 1) can be accessed an equals 0
+//! @param  n   size of matrix
+//! @param  lg  lower bound of input interval (e.g. Gerschgorin interval)
+//! @param  ug  upper bound of input interval (e.g. Gerschgorin interval)
+//! @param  lg_eig_count  number of eigenvalues that are smaller than \a lg
+//! @param  lu_eig_count  number of eigenvalues that are smaller than \a lu
+//! @param  epsilon  desired accuracy of eigenvalues to compute
+////////////////////////////////////////////////////////////////////////////////
+__global__ void bisectKernel(float             *g_d,
+                             float             *g_s,
+                             const unsigned int n,
+                             float             *g_left,
+                             float             *g_right,
+                             unsigned int      *g_left_count,
+                             unsigned int      *g_right_count,
+                             const float        lg,
+                             const float        ug,
+                             const unsigned int lg_eig_count,
+                             const unsigned int ug_eig_count,
+                             float              epsilon)
+{
+    // Handle to thread block group
+    cg::thread_block cta = cg::this_thread_block();
+    // intervals (store left and right because the subdivision tree is in general
+    // not dense
+    __shared__ float s_left[MAX_THREADS_BLOCK_SMALL_MATRIX];
+    __shared__ float s_right[MAX_THREADS_BLOCK_SMALL_MATRIX];
+
+    // number of eigenvalues that are smaller than s_left / s_right
+    // (correspondence is realized via indices)
+    __shared__ unsigned int s_left_count[MAX_THREADS_BLOCK_SMALL_MATRIX];
+    __shared__ unsigned int s_right_count[MAX_THREADS_BLOCK_SMALL_MATRIX];
+
+    // helper for stream compaction
+    __shared__ unsigned int s_compaction_list[MAX_THREADS_BLOCK_SMALL_MATRIX + 1];
+
+    // state variables for whole block
+    // if 0 then compaction of second chunk of child intervals is not necessary
+    // (because all intervals had exactly one non-dead child)
+    __shared__ unsigned int compact_second_chunk;
+    __shared__ unsigned int all_threads_converged;
+
+    // number of currently active threads
+    __shared__ unsigned int num_threads_active;
+
+    // number of threads to use for stream compaction
+    __shared__ unsigned int num_threads_compaction;
+
+    // helper for exclusive scan
+    unsigned int *s_compaction_list_exc = s_compaction_list + 1;
+
+    // variables for currently processed interval
+    // left and right limit of active interval
+    float        left        = 0.0f;
+    float        right       = 0.0f;
+    unsigned int left_count  = 0;
+    unsigned int right_count = 0;
+    // midpoint of active interval
+    float mid = 0.0f;
+    // number of eigenvalues smaller then mid
+    unsigned int mid_count = 0;
+    // affected from compaction
+    unsigned int is_active_second = 0;
+
+    s_compaction_list[threadIdx.x] = 0;
+    s_left[threadIdx.x]            = 0;
+    s_right[threadIdx.x]           = 0;
+    s_left_count[threadIdx.x]      = 0;
+    s_right_count[threadIdx.x]     = 0;
+
+    cg::sync(cta);
+
+    // set up initial configuration
+    if (0 == threadIdx.x) {
+        s_left[0]        = lg;
+        s_right[0]       = ug;
+        s_left_count[0]  = lg_eig_count;
+        s_right_count[0] = ug_eig_count;
+
+        compact_second_chunk = 0;
+        num_threads_active   = 1;
+
+        num_threads_compaction = 1;
+    }
+
+    // for all active threads read intervals from the last level
+    // the number of (worst case) active threads per level l is 2^l
+    while (true) {
+        all_threads_converged = 1;
+        cg::sync(cta);
+
+        is_active_second = 0;
+        subdivideActiveInterval(threadIdx.x,
+                                s_left,
+                                s_right,
+                                s_left_count,
+                                s_right_count,
+                                num_threads_active,
+                                left,
+                                right,
+                                left_count,
+                                right_count,
+                                mid,
+                                all_threads_converged);
+
+        cg::sync(cta);
+
+        // check if done
+        if (1 == all_threads_converged) {
+            break;
+        }
+
+        cg::sync(cta);
+
+        // compute number of eigenvalues smaller than mid
+        // use all threads for reading the necessary matrix data from global
+        // memory
+        // use s_left and s_right as scratch space for diagonal and
+        // superdiagonal of matrix
+        mid_count = computeNumSmallerEigenvals(
+            g_d, g_s, n, mid, threadIdx.x, num_threads_active, s_left, s_right, (left == right), cta);
+
+        cg::sync(cta);
+
+        // store intervals
+        // for all threads store the first child interval in a continuous chunk of
+        // memory, and the second child interval -- if it exists -- in a second
+        // chunk; it is likely that all threads reach convergence up to
+        // \a epsilon at the same level; furthermore, for higher level most / all
+        // threads will have only one child, storing the first child compactly will
+        // (first) avoid to perform a compaction step on the first chunk, (second)
+        // make it for higher levels (when all threads / intervals have
+        // exactly one child)  unnecessary to perform a compaction of the second
+        // chunk
+        if (threadIdx.x < num_threads_active) {
+            if (left != right) {
+                // store intervals
+                storeNonEmptyIntervals(threadIdx.x,
+                                       num_threads_active,
+                                       s_left,
+                                       s_right,
+                                       s_left_count,
+                                       s_right_count,
+                                       left,
+                                       mid,
+                                       right,
+                                       left_count,
+                                       mid_count,
+                                       right_count,
+                                       epsilon,
+                                       compact_second_chunk,
+                                       s_compaction_list_exc,
+                                       is_active_second);
+            }
+            else {
+                storeIntervalConverged(s_left,
+                                       s_right,
+                                       s_left_count,
+                                       s_right_count,
+                                       left,
+                                       mid,
+                                       right,
+                                       left_count,
+                                       mid_count,
+                                       right_count,
+                                       s_compaction_list_exc,
+                                       compact_second_chunk,
+                                       num_threads_active,
+                                       is_active_second);
+            }
+        }
+
+        // necessary so that compact_second_chunk is up-to-date
+        cg::sync(cta);
+
+        // perform compaction of chunk where second children are stored
+        // scan of (num_threads_active / 2) elements, thus at most
+        // (num_threads_active / 4) threads are needed
+        if (compact_second_chunk > 0) {
+            createIndicesCompaction(s_compaction_list_exc, num_threads_compaction, cta);
+
+            compactIntervals(s_left,
+                             s_right,
+                             s_left_count,
+                             s_right_count,
+                             mid,
+                             right,
+                             mid_count,
+                             right_count,
+                             s_compaction_list,
+                             num_threads_active,
+                             is_active_second);
+        }
+
+        cg::sync(cta);
+
+        if (0 == threadIdx.x) {
+            // update number of active threads with result of reduction
+            num_threads_active += s_compaction_list[num_threads_active];
+
+            num_threads_compaction = ceilPow2(num_threads_active);
+
+            compact_second_chunk = 0;
+        }
+
+        cg::sync(cta);
+    }
+
+    cg::sync(cta);
+
+    // write resulting intervals to global mem
+    // for all threads write if they have been converged to an eigenvalue to
+    // a separate array
+
+    // at most n valid intervals
+    if (threadIdx.x < n) {
+        // intervals converged so left and right limit are identical
+        g_left[threadIdx.x] = s_left[threadIdx.x];
+        // left count is sufficient to have global order
+        g_left_count[threadIdx.x] = s_left_count[threadIdx.x];
+    }
+}
+
+#endif // #ifndef _BISECT_KERNEL_SMALL_H_

cpp/2_Concepts_and_Techniques/eigenvalues/bisect_large.cu

@@ -0,0 +1,368 @@
+/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/* Computation of eigenvalues of a large symmetric, tridiagonal matrix */
+
+// includes, system
+#include <float.h>
+#include <math.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+// includes, project
+#include "bisect_large.cuh"
+#include "config.h"
+#include "helper_cuda.h"
+#include "helper_functions.h"
+#include "matlab.h"
+#include "structs.h"
+#include "util.h"
+
+// includes, kernels
+#include "bisect_kernel_large.cuh"
+#include "bisect_kernel_large_multi.cuh"
+#include "bisect_kernel_large_onei.cuh"
+
+////////////////////////////////////////////////////////////////////////////////
+//! Initialize variables and memory for result
+//! @param  result handles to memory
+//! @param  matrix_size  size of the matrix
+////////////////////////////////////////////////////////////////////////////////
+void initResultDataLargeMatrix(ResultDataLarge &result, const unsigned int mat_size)
+{
+    // helper variables to initialize memory
+    unsigned int zero        = 0;
+    unsigned int mat_size_f  = sizeof(float) * mat_size;
+    unsigned int mat_size_ui = sizeof(unsigned int) * mat_size;
+
+    float        *tempf  = (float *)malloc(mat_size_f);
+    unsigned int *tempui = (unsigned int *)malloc(mat_size_ui);
+
+    for (unsigned int i = 0; i < mat_size; ++i) {
+        tempf[i]  = 0.0f;
+        tempui[i] = 0;
+    }
+
+    // number of intervals containing only one eigenvalue after the first step
+    checkCudaErrors(cudaMalloc((void **)&result.g_num_one, sizeof(unsigned int)));
+    checkCudaErrors(cudaMemcpy(result.g_num_one, &zero, sizeof(unsigned int), cudaMemcpyHostToDevice));
+
+    // number of (thread) blocks of intervals with multiple eigenvalues after
+    // the first iteration
+    checkCudaErrors(cudaMalloc((void **)&result.g_num_blocks_mult, sizeof(unsigned int)));
+    checkCudaErrors(cudaMemcpy(result.g_num_blocks_mult, &zero, sizeof(unsigned int), cudaMemcpyHostToDevice));
+
+    checkCudaErrors(cudaMalloc((void **)&result.g_left_one, mat_size_f));
+    checkCudaErrors(cudaMalloc((void **)&result.g_right_one, mat_size_f));
+    checkCudaErrors(cudaMalloc((void **)&result.g_pos_one, mat_size_ui));
+
+    checkCudaErrors(cudaMalloc((void **)&result.g_left_mult, mat_size_f));
+    checkCudaErrors(cudaMalloc((void **)&result.g_right_mult, mat_size_f));
+    checkCudaErrors(cudaMalloc((void **)&result.g_left_count_mult, mat_size_ui));
+    checkCudaErrors(cudaMalloc((void **)&result.g_right_count_mult, mat_size_ui));
+
+    checkCudaErrors(cudaMemcpy(result.g_left_one, tempf, mat_size_f, cudaMemcpyHostToDevice));
+    checkCudaErrors(cudaMemcpy(result.g_right_one, tempf, mat_size_f, cudaMemcpyHostToDevice));
+    checkCudaErrors(cudaMemcpy(result.g_pos_one, tempui, mat_size_ui, cudaMemcpyHostToDevice));
+
+    checkCudaErrors(cudaMemcpy(result.g_left_mult, tempf, mat_size_f, cudaMemcpyHostToDevice));
+    checkCudaErrors(cudaMemcpy(result.g_right_mult, tempf, mat_size_f, cudaMemcpyHostToDevice));
+    checkCudaErrors(cudaMemcpy(result.g_left_count_mult, tempui, mat_size_ui, cudaMemcpyHostToDevice));
+    checkCudaErrors(cudaMemcpy(result.g_right_count_mult, tempui, mat_size_ui, cudaMemcpyHostToDevice));
+
+    checkCudaErrors(cudaMalloc((void **)&result.g_blocks_mult, mat_size_ui));
+    checkCudaErrors(cudaMemcpy(result.g_blocks_mult, tempui, mat_size_ui, cudaMemcpyHostToDevice));
+    checkCudaErrors(cudaMalloc((void **)&result.g_blocks_mult_sum, mat_size_ui));
+    checkCudaErrors(cudaMemcpy(result.g_blocks_mult_sum, tempui, mat_size_ui, cudaMemcpyHostToDevice));
+
+    checkCudaErrors(cudaMalloc((void **)&result.g_lambda_mult, mat_size_f));
+    checkCudaErrors(cudaMemcpy(result.g_lambda_mult, tempf, mat_size_f, cudaMemcpyHostToDevice));
+    checkCudaErrors(cudaMalloc((void **)&result.g_pos_mult, mat_size_ui));
+    checkCudaErrors(cudaMemcpy(result.g_pos_mult, tempf, mat_size_ui, cudaMemcpyHostToDevice));
+}
+
+////////////////////////////////////////////////////////////////////////////////
+//! Cleanup result memory
+//! @param result  handles to memory
+////////////////////////////////////////////////////////////////////////////////
+void cleanupResultDataLargeMatrix(ResultDataLarge &result)
+{
+    checkCudaErrors(cudaFree(result.g_num_one));
+    checkCudaErrors(cudaFree(result.g_num_blocks_mult));
+    checkCudaErrors(cudaFree(result.g_left_one));
+    checkCudaErrors(cudaFree(result.g_right_one));
+    checkCudaErrors(cudaFree(result.g_pos_one));
+    checkCudaErrors(cudaFree(result.g_left_mult));
+    checkCudaErrors(cudaFree(result.g_right_mult));
+    checkCudaErrors(cudaFree(result.g_left_count_mult));
+    checkCudaErrors(cudaFree(result.g_right_count_mult));
+    checkCudaErrors(cudaFree(result.g_blocks_mult));
+    checkCudaErrors(cudaFree(result.g_blocks_mult_sum));
+    checkCudaErrors(cudaFree(result.g_lambda_mult));
+    checkCudaErrors(cudaFree(result.g_pos_mult));
+}
+
+////////////////////////////////////////////////////////////////////////////////
+//! Run the kernels to compute the eigenvalues for large matrices
+//! @param  input   handles to input data
+//! @param  result  handles to result data
+//! @param  mat_size  matrix size
+//! @param  precision  desired precision of eigenvalues
+//! @param  lg  lower limit of Gerschgorin interval
+//! @param  ug  upper limit of Gerschgorin interval
+//! @param  iterations  number of iterations (for timing)
+////////////////////////////////////////////////////////////////////////////////
+void computeEigenvaluesLargeMatrix(const InputData       &input,
+                                   const ResultDataLarge &result,
+                                   const unsigned int     mat_size,
+                                   const float            precision,
+                                   const float            lg,
+                                   const float            ug,
+                                   const unsigned int     iterations)
+{
+    dim3 blocks(1, 1, 1);
+    dim3 threads(MAX_THREADS_BLOCK, 1, 1);
+
+    StopWatchInterface *timer_step1      = NULL;
+    StopWatchInterface *timer_step2_one  = NULL;
+    StopWatchInterface *timer_step2_mult = NULL;
+    StopWatchInterface *timer_total      = NULL;
+    sdkCreateTimer(&timer_step1);
+    sdkCreateTimer(&timer_step2_one);
+    sdkCreateTimer(&timer_step2_mult);
+    sdkCreateTimer(&timer_total);
+
+    sdkStartTimer(&timer_total);
+
+    // do for multiple iterations to improve timing accuracy
+    for (unsigned int iter = 0; iter < iterations; ++iter) {
+        sdkStartTimer(&timer_step1);
+        bisectKernelLarge<<<blocks, threads>>>(input.g_a,
+                                               input.g_b,
+                                               mat_size,
+                                               lg,
+                                               ug,
+                                               0,
+                                               mat_size,
+                                               precision,
+                                               result.g_num_one,
+                                               result.g_num_blocks_mult,
+                                               result.g_left_one,
+                                               result.g_right_one,
+                                               result.g_pos_one,
+                                               result.g_left_mult,
+                                               result.g_right_mult,
+                                               result.g_left_count_mult,
+                                               result.g_right_count_mult,
+                                               result.g_blocks_mult,
+                                               result.g_blocks_mult_sum);
+
+        getLastCudaError("Kernel launch failed.");
+        checkCudaErrors(cudaDeviceSynchronize());
+        sdkStopTimer(&timer_step1);
+
+        // get the number of intervals containing one eigenvalue after the first
+        // processing step
+        unsigned int num_one_intervals;
+        checkCudaErrors(cudaMemcpy(&num_one_intervals, result.g_num_one, sizeof(unsigned int), cudaMemcpyDeviceToHost));
+
+        dim3 grid_onei;
+        grid_onei.x = getNumBlocksLinear(num_one_intervals, MAX_THREADS_BLOCK);
+        dim3 threads_onei;
+        // use always max number of available threads to better balance load times
+        // for matrix data
+        threads_onei.x = MAX_THREADS_BLOCK;
+
+        // compute eigenvalues for intervals that contained only one eigenvalue
+        // after the first processing step
+        sdkStartTimer(&timer_step2_one);
+
+        bisectKernelLarge_OneIntervals<<<grid_onei, threads_onei>>>(input.g_a,
+                                                                    input.g_b,
+                                                                    mat_size,
+                                                                    num_one_intervals,
+                                                                    result.g_left_one,
+                                                                    result.g_right_one,
+                                                                    result.g_pos_one,
+                                                                    precision);
+
+        getLastCudaError("bisectKernelLarge_OneIntervals() FAILED.");
+        checkCudaErrors(cudaDeviceSynchronize());
+        sdkStopTimer(&timer_step2_one);
+
+        // process intervals that contained more than one eigenvalue after
+        // the first processing step
+
+        // get the number of blocks of intervals that contain, in total when
+        // each interval contains only one eigenvalue, not more than
+        // MAX_THREADS_BLOCK threads
+        unsigned int num_blocks_mult = 0;
+        checkCudaErrors(
+            cudaMemcpy(&num_blocks_mult, result.g_num_blocks_mult, sizeof(unsigned int), cudaMemcpyDeviceToHost));
+
+        // setup the execution environment
+        dim3 grid_mult(num_blocks_mult, 1, 1);
+        dim3 threads_mult(MAX_THREADS_BLOCK, 1, 1);
+
+        sdkStartTimer(&timer_step2_mult);
+
+        bisectKernelLarge_MultIntervals<<<grid_mult, threads_mult>>>(input.g_a,
+                                                                     input.g_b,
+                                                                     mat_size,
+                                                                     result.g_blocks_mult,
+                                                                     result.g_blocks_mult_sum,
+                                                                     result.g_left_mult,
+                                                                     result.g_right_mult,
+                                                                     result.g_left_count_mult,
+                                                                     result.g_right_count_mult,
+                                                                     result.g_lambda_mult,
+                                                                     result.g_pos_mult,
+                                                                     precision);
+
+        getLastCudaError("bisectKernelLarge_MultIntervals() FAILED.");
+        checkCudaErrors(cudaDeviceSynchronize());
+        sdkStopTimer(&timer_step2_mult);
+    }
+
+    sdkStopTimer(&timer_total);
+
+    printf("Average time step 1: %f ms\n", sdkGetTimerValue(&timer_step1) / (float)iterations);
+    printf("Average time step 2, one intervals: %f ms\n", sdkGetTimerValue(&timer_step2_one) / (float)iterations);
+    printf("Average time step 2, mult intervals: %f ms\n", sdkGetTimerValue(&timer_step2_mult) / (float)iterations);
+
+    printf("Average time TOTAL: %f ms\n", sdkGetTimerValue(&timer_total) / (float)iterations);
+
+    sdkDeleteTimer(&timer_step1);
+    sdkDeleteTimer(&timer_step2_one);
+    sdkDeleteTimer(&timer_step2_mult);
+    sdkDeleteTimer(&timer_total);
+}
+
+////////////////////////////////////////////////////////////////////////////////
+//! Process the result, that is obtain result from device and do simple sanity
+//! checking
+//! @param  input   handles to input data
+//! @param  result  handles to result data
+//! @param  mat_size  matrix size
+//! @param  filename  output filename
+////////////////////////////////////////////////////////////////////////////////
+bool processResultDataLargeMatrix(const InputData       &input,
+                                  const ResultDataLarge &result,
+                                  const unsigned int     mat_size,
+                                  const char            *filename,
+                                  const unsigned int     user_defined,
+                                  char                  *exec_path)
+{
+    bool               bCompareResult = false;
+    const unsigned int mat_size_ui    = sizeof(unsigned int) * mat_size;
+    const unsigned int mat_size_f     = sizeof(float) * mat_size;
+
+    // copy data from intervals that contained more than one eigenvalue after
+    // the first processing step
+    float *lambda_mult = (float *)malloc(sizeof(float) * mat_size);
+    checkCudaErrors(cudaMemcpy(lambda_mult, result.g_lambda_mult, sizeof(float) * mat_size, cudaMemcpyDeviceToHost));
+    unsigned int *pos_mult = (unsigned int *)malloc(sizeof(unsigned int) * mat_size);
+    checkCudaErrors(cudaMemcpy(pos_mult, result.g_pos_mult, sizeof(unsigned int) * mat_size, cudaMemcpyDeviceToHost));
+
+    unsigned int *blocks_mult_sum = (unsigned int *)malloc(sizeof(unsigned int) * mat_size);
+    checkCudaErrors(
+        cudaMemcpy(blocks_mult_sum, result.g_blocks_mult_sum, sizeof(unsigned int) * mat_size, cudaMemcpyDeviceToHost));
+
+    unsigned int num_one_intervals;
+    checkCudaErrors(cudaMemcpy(&num_one_intervals, result.g_num_one, sizeof(unsigned int), cudaMemcpyDeviceToHost));
+
+    unsigned int sum_blocks_mult = mat_size - num_one_intervals;
+
+    // copy data for intervals that contained one eigenvalue after the first
+    // processing step
+    float        *left_one  = (float *)malloc(mat_size_f);
+    float        *right_one = (float *)malloc(mat_size_f);
+    unsigned int *pos_one   = (unsigned int *)malloc(mat_size_ui);
+    checkCudaErrors(cudaMemcpy(left_one, result.g_left_one, mat_size_f, cudaMemcpyDeviceToHost));
+    checkCudaErrors(cudaMemcpy(right_one, result.g_right_one, mat_size_f, cudaMemcpyDeviceToHost));
+    checkCudaErrors(cudaMemcpy(pos_one, result.g_pos_one, mat_size_ui, cudaMemcpyDeviceToHost));
+
+    // extract eigenvalues
+    float *eigenvals = (float *)malloc(mat_size_f);
+
+    // singleton intervals generated in the second step
+    for (unsigned int i = 0; i < sum_blocks_mult; ++i) {
+        eigenvals[pos_mult[i] - 1] = lambda_mult[i];
+    }
+
+    // singleton intervals generated in the first step
+    unsigned int index = 0;
+
+    for (unsigned int i = 0; i < num_one_intervals; ++i, ++index) {
+        eigenvals[pos_one[i] - 1] = left_one[i];
+    }
+
+    if (1 == user_defined) {
+        // store result
+        writeTridiagSymMatlab(filename, input.a, input.b + 1, eigenvals, mat_size);
+        // getLastCudaError( sdkWriteFilef( filename, eigenvals, mat_size, 0.0f));
+
+        printf("User requests non-default argument(s), skipping self-check!\n");
+        bCompareResult = true;
+    }
+    else {
+        // compare with reference solution
+
+        float       *reference       = NULL;
+        unsigned int input_data_size = 0;
+
+        char *ref_path = sdkFindFilePath("reference.dat", exec_path);
+        assert(NULL != ref_path);
+        sdkReadFile(ref_path, &reference, &input_data_size, false);
+        assert(input_data_size == mat_size);
+
+        // there's an imprecision of Sturm count computation which makes an
+        // additional offset necessary
+        float tolerance = 1.0e-5f + 5.0e-6f;
+
+        if (sdkCompareL2fe(reference, eigenvals, mat_size, tolerance) == true) {
+            bCompareResult = true;
+        }
+        else {
+            bCompareResult = false;
+        }
+
+        free(ref_path);
+        free(reference);
+    }
+
+    freePtr(eigenvals);
+    freePtr(lambda_mult);
+    freePtr(pos_mult);
+    freePtr(blocks_mult_sum);
+    freePtr(left_one);
+    freePtr(right_one);
+    freePtr(pos_one);
+
+    return bCompareResult;
+}

cpp/2_Concepts_and_Techniques/eigenvalues/bisect_large.cuh

@@ -0,0 +1,85 @@
+/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/* Computation of eigenvalues of a small bidiagonal matrix */
+
+#ifndef _BISECT_LARGE_CUH_
+#define _BISECT_LARGE_CUH_
+
+#include "structs.h"
+
+extern "C"
+{
+
+    ////////////////////////////////////////////////////////////////////////////////
+    //! Run the kernels to compute the eigenvalues for large matrices
+    //! @param  input   handles to input data
+    //! @param  result  handles to result data
+    //! @param  mat_size  matrix size
+    //! @param  precision  desired precision of eigenvalues
+    //! @param  lg  lower limit of Gerschgorin interval
+    //! @param  ug  upper limit of Gerschgorin interval
+    //! @param  iterations  number of iterations (for timing)
+    ////////////////////////////////////////////////////////////////////////////////
+    void computeEigenvaluesLargeMatrix(const InputData       &input,
+                                       const ResultDataLarge &result,
+                                       const unsigned int     mat_size,
+                                       const float            precision,
+                                       const float            lg,
+                                       const float            ug,
+                                       const unsigned int     iterations);
+
+    ////////////////////////////////////////////////////////////////////////////////
+    //! Initialize variables and memory for result
+    //! @param  result handles to memory
+    //! @param  matr_size  size of the matrix
+    ////////////////////////////////////////////////////////////////////////////////
+    void initResultDataLargeMatrix(ResultDataLarge &result, const unsigned int mat_size);
+
+    ////////////////////////////////////////////////////////////////////////////////
+    //! Cleanup result memory
+    //! @param result  handles to memory
+    ////////////////////////////////////////////////////////////////////////////////
+    void cleanupResultDataLargeMatrix(ResultDataLarge &result);
+
+    ////////////////////////////////////////////////////////////////////////////////
+    //! Process the result, that is obtain result from device and do simple sanity
+    //! checking
+    //! @param  input   handles to input data
+    //! @param  result  handles to result data
+    //! @param  mat_size  matrix size
+    //! @param  filename  output filename
+    ////////////////////////////////////////////////////////////////////////////////
+    bool processResultDataLargeMatrix(const InputData       &input,
+                                      const ResultDataLarge &result,
+                                      const unsigned int     mat_size,
+                                      const char            *filename,
+                                      const unsigned int     user_defined,
+                                      char                  *exec_path);
+};
+
+#endif // #ifndef _BISECT_LARGE_CUH_

cpp/2_Concepts_and_Techniques/eigenvalues/bisect_small.cu

@@ -0,0 +1,186 @@
+/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/* Computation of eigenvalues of a small symmetric, tridiagonal matrix */
+
+// includes, system
+#include <float.h>
+#include <math.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+// includes, project
+#include "config.h"
+#include "helper_cuda.h"
+#include "helper_functions.h"
+#include "matlab.h"
+#include "structs.h"
+
+// includes, kernels
+#include "bisect_kernel_small.cuh"
+
+// includes, file
+#include "bisect_small.cuh"
+
+////////////////////////////////////////////////////////////////////////////////
+//! Determine eigenvalues for matrices smaller than MAX_SMALL_MATRIX
+//! @param TimingIterations  number of iterations for timing
+//! @param  input  handles to input data of kernel
+//! @param  result handles to result of kernel
+//! @param  mat_size  matrix size
+//! @param  lg  lower limit of Gerschgorin interval
+//! @param  ug  upper limit of Gerschgorin interval
+//! @param  precision  desired precision of eigenvalues
+//! @param  iterations  number of iterations for timing
+////////////////////////////////////////////////////////////////////////////////
+void computeEigenvaluesSmallMatrix(const InputData   &input,
+                                   ResultDataSmall   &result,
+                                   const unsigned int mat_size,
+                                   const float        lg,
+                                   const float        ug,
+                                   const float        precision,
+                                   const unsigned int iterations)
+{
+    StopWatchInterface *timer = NULL;
+    sdkCreateTimer(&timer);
+    sdkStartTimer(&timer);
+
+    for (unsigned int i = 0; i < iterations; ++i) {
+        dim3 blocks(1, 1, 1);
+        dim3 threads(MAX_THREADS_BLOCK_SMALL_MATRIX, 1, 1);
+
+        bisectKernel<<<blocks, threads>>>(input.g_a,
+                                          input.g_b,
+                                          mat_size,
+                                          result.g_left,
+                                          result.g_right,
+                                          result.g_left_count,
+                                          result.g_right_count,
+                                          lg,
+                                          ug,
+                                          0,
+                                          mat_size,
+                                          precision);
+    }
+
+    checkCudaErrors(cudaDeviceSynchronize());
+    sdkStopTimer(&timer);
+    getLastCudaError("Kernel launch failed");
+    printf("Average time: %f ms (%i iterations)\n", sdkGetTimerValue(&timer) / (float)iterations, iterations);
+
+    sdkDeleteTimer(&timer);
+}
+
+////////////////////////////////////////////////////////////////////////////////
+//! Initialize variables and memory for the result for small matrices
+//! @param result  handles to the necessary memory
+//! @param  mat_size  matrix_size
+////////////////////////////////////////////////////////////////////////////////
+void initResultSmallMatrix(ResultDataSmall &result, const unsigned int mat_size)
+{
+    result.mat_size_f  = sizeof(float) * mat_size;
+    result.mat_size_ui = sizeof(unsigned int) * mat_size;
+
+    result.eigenvalues = (float *)malloc(result.mat_size_f);
+
+    // helper variables
+    result.zero_f  = (float *)malloc(result.mat_size_f);
+    result.zero_ui = (unsigned int *)malloc(result.mat_size_ui);
+
+    for (unsigned int i = 0; i < mat_size; ++i) {
+        result.zero_f[i]  = 0.0f;
+        result.zero_ui[i] = 0;
+
+        result.eigenvalues[i] = 0.0f;
+    }
+
+    checkCudaErrors(cudaMalloc((void **)&result.g_left, result.mat_size_f));
+    checkCudaErrors(cudaMalloc((void **)&result.g_right, result.mat_size_f));
+
+    checkCudaErrors(cudaMalloc((void **)&result.g_left_count, result.mat_size_ui));
+    checkCudaErrors(cudaMalloc((void **)&result.g_right_count, result.mat_size_ui));
+
+    // initialize result memory
+    checkCudaErrors(cudaMemcpy(result.g_left, result.zero_f, result.mat_size_f, cudaMemcpyHostToDevice));
+    checkCudaErrors(cudaMemcpy(result.g_right, result.zero_f, result.mat_size_f, cudaMemcpyHostToDevice));
+    checkCudaErrors(cudaMemcpy(result.g_right_count, result.zero_ui, result.mat_size_ui, cudaMemcpyHostToDevice));
+    checkCudaErrors(cudaMemcpy(result.g_left_count, result.zero_ui, result.mat_size_ui, cudaMemcpyHostToDevice));
+}
+
+////////////////////////////////////////////////////////////////////////////////
+//! Cleanup memory and variables for result for small matrices
+//! @param  result  handle to variables
+////////////////////////////////////////////////////////////////////////////////
+void cleanupResultSmallMatrix(ResultDataSmall &result)
+{
+    freePtr(result.eigenvalues);
+    freePtr(result.zero_f);
+    freePtr(result.zero_ui);
+
+    checkCudaErrors(cudaFree(result.g_left));
+    checkCudaErrors(cudaFree(result.g_right));
+    checkCudaErrors(cudaFree(result.g_left_count));
+    checkCudaErrors(cudaFree(result.g_right_count));
+}
+
+////////////////////////////////////////////////////////////////////////////////
+//! Process the result obtained on the device, that is transfer to host and
+//! perform basic sanity checking
+//! @param  input  handles to input data
+//! @param  result  handles to result data
+//! @param  mat_size   matrix size
+//! @param  filename  output filename
+////////////////////////////////////////////////////////////////////////////////
+void processResultSmallMatrix(const InputData       &input,
+                              const ResultDataSmall &result,
+                              const unsigned int     mat_size,
+                              const char            *filename)
+{
+    const unsigned int mat_size_f  = sizeof(float) * mat_size;
+    const unsigned int mat_size_ui = sizeof(unsigned int) * mat_size;
+
+    // copy data back to host
+    float        *left       = (float *)malloc(mat_size_f);
+    unsigned int *left_count = (unsigned int *)malloc(mat_size_ui);
+
+    checkCudaErrors(cudaMemcpy(left, result.g_left, mat_size_f, cudaMemcpyDeviceToHost));
+    checkCudaErrors(cudaMemcpy(left_count, result.g_left_count, mat_size_ui, cudaMemcpyDeviceToHost));
+
+    float *eigenvalues = (float *)malloc(mat_size_f);
+
+    for (unsigned int i = 0; i < mat_size; ++i) {
+        eigenvalues[left_count[i]] = left[i];
+    }
+
+    // save result in matlab format
+    writeTridiagSymMatlab(filename, input.a, input.b + 1, eigenvalues, mat_size);
+
+    freePtr(left);
+    freePtr(left_count);
+    freePtr(eigenvalues);
+}

cpp/2_Concepts_and_Techniques/eigenvalues/bisect_small.cuh

@@ -0,0 +1,82 @@
+/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/* Computation of eigenvalues of a small bidiagonal matrix */
+
+#ifndef _BISECT_SMALL_CUH_
+#define _BISECT_SMALL_CUH_
+
+extern "C"
+{
+
+    ////////////////////////////////////////////////////////////////////////////////
+    //! Determine eigenvalues for matrices smaller than MAX_SMALL_MATRIX
+    //! @param TimingIterations  number of iterations for timing
+    //! @param  input  handles to input data of kernel
+    //! @param  result handles to result of kernel
+    //! @param  mat_size  matrix size
+    //! @param  lg  lower limit of Gerschgorin interval
+    //! @param  ug  upper limit of Gerschgorin interval
+    //! @param  precision  desired precision of eigenvalues
+    //! @param  iterations  number of iterations for timing
+    ////////////////////////////////////////////////////////////////////////////////
+    void computeEigenvaluesSmallMatrix(const InputData   &input,
+                                       ResultDataSmall   &result,
+                                       const unsigned int mat_size,
+                                       const float        lg,
+                                       const float        ug,
+                                       const float        precision,
+                                       const unsigned int iterations);
+
+    ////////////////////////////////////////////////////////////////////////////////
+    //! Initialize variables and memory for the result for small matrices
+    //! @param result  handles to the necessary memory
+    //! @param  mat_size  matrix_size
+    ////////////////////////////////////////////////////////////////////////////////
+    void initResultSmallMatrix(ResultDataSmall &result, const unsigned int mat_size);
+
+    ////////////////////////////////////////////////////////////////////////////////
+    //! Cleanup memory and variables for result for small matrices
+    //! @param  result  handle to variables
+    ////////////////////////////////////////////////////////////////////////////////
+    void cleanupResultSmallMatrix(ResultDataSmall &result);
+
+    ////////////////////////////////////////////////////////////////////////////////
+    //! Process the result obtained on the device, that is transfer to host and
+    //! perform basic sanity checking
+    //! @param  input   handles to input data
+    //! @param  result  handles to result variables
+    //! @param  mat_size   matrix size
+    //! @param  filename  output filename
+    ////////////////////////////////////////////////////////////////////////////////
+    void processResultSmallMatrix(const InputData       &input,
+                                  const ResultDataSmall &result,
+                                  const unsigned int     mat_size,
+                                  const char            *filename);
+}
+
+#endif // #ifndef _BISECT_SMALL_CUH_

cpp/2_Concepts_and_Techniques/eigenvalues/bisect_util.cu

@@ -0,0 +1,596 @@
+/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/* Utility / shared functionality for bisection kernels */
+
+#ifndef _BISECT_UTIL_H_
+#define _BISECT_UTIL_H_
+
+#include <cooperative_groups.h>
+
+namespace cg = cooperative_groups;
+
+// includes, project
+#include "config.h"
+#include "util.h"
+
+////////////////////////////////////////////////////////////////////////////////
+//! Compute the next lower power of two of n
+//! @param  n  number for which next higher power of two is sought
+////////////////////////////////////////////////////////////////////////////////
+__device__ inline int floorPow2(int n)
+{
+    // early out if already power of two
+    if (0 == (n & (n - 1))) {
+        return n;
+    }
+
+    int exp;
+    frexp((float)n, &exp);
+    return (1 << (exp - 1));
+}
+
+////////////////////////////////////////////////////////////////////////////////
+//! Compute the next higher power of two of n
+//! @param  n  number for which next higher power of two is sought
+////////////////////////////////////////////////////////////////////////////////
+__device__ inline int ceilPow2(int n)
+{
+    // early out if already power of two
+    if (0 == (n & (n - 1))) {
+        return n;
+    }
+
+    int exp;
+    frexp((float)n, &exp);
+    return (1 << exp);
+}
+
+////////////////////////////////////////////////////////////////////////////////
+//! Compute midpoint of interval [\a left, \a right] avoiding overflow if
+//! possible
+//! @param left   left / lower limit of interval
+//! @param right  right / upper limit of interval
+////////////////////////////////////////////////////////////////////////////////
+__device__ inline float computeMidpoint(const float left, const float right)
+{
+    float mid;
+
+    if (sign_f(left) == sign_f(right)) {
+        mid = left + (right - left) * 0.5f;
+    }
+    else {
+        mid = (left + right) * 0.5f;
+    }
+
+    return mid;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+//! Check if interval converged and store appropriately
+//! @param  addr    address where to store the information of the interval
+//! @param  s_left  shared memory storage for left interval limits
+//! @param  s_right  shared memory storage for right interval limits
+//! @param  s_left_count  shared memory storage for number of eigenvalues less
+//!                       than left interval limits
+//! @param  s_right_count  shared memory storage for number of eigenvalues less
+//!                       than right interval limits
+//! @param  left   lower limit of interval
+//! @param  right  upper limit of interval
+//! @param  left_count  eigenvalues less than \a left
+//! @param  right_count  eigenvalues less than \a right
+//! @param  precision  desired precision for eigenvalues
+////////////////////////////////////////////////////////////////////////////////
+template <class S, class T>
+__device__ void storeInterval(unsigned int addr,
+                              float       *s_left,
+                              float       *s_right,
+                              T           *s_left_count,
+                              T           *s_right_count,
+                              float        left,
+                              float        right,
+                              S            left_count,
+                              S            right_count,
+                              float        precision)
+{
+    s_left_count[addr]  = left_count;
+    s_right_count[addr] = right_count;
+
+    // check if interval converged
+    float t0 = abs(right - left);
+    float t1 = max(abs(left), abs(right)) * precision;
+
+    if (t0 <= max(MIN_ABS_INTERVAL, t1)) {
+        // compute mid point
+        float lambda = computeMidpoint(left, right);
+
+        // mark as converged
+        s_left[addr]  = lambda;
+        s_right[addr] = lambda;
+    }
+    else {
+        // store current limits
+        s_left[addr]  = left;
+        s_right[addr] = right;
+    }
+}
+
+////////////////////////////////////////////////////////////////////////////////
+//! Compute number of eigenvalues that are smaller than x given a symmetric,
+//! real, and tridiagonal matrix
+//! @param  g_d  diagonal elements stored in global memory
+//! @param  g_s  superdiagonal elements stored in global memory
+//! @param  n    size of matrix
+//! @param  x    value for which the number of eigenvalues that are smaller is
+//!              seeked
+//! @param  tid  thread identified (e.g. threadIdx.x or gtid)
+//! @param  num_intervals_active  number of active intervals / threads that
+//!                               currently process an interval
+//! @param  s_d  scratch space to store diagonal entries of the tridiagonal
+//!              matrix in shared memory
+//! @param  s_s  scratch space to store superdiagonal entries of the tridiagonal
+//!              matrix in shared memory
+//! @param  converged  flag if the current thread is already converged (that
+//!         is count does not have to be computed)
+////////////////////////////////////////////////////////////////////////////////
+__device__ inline unsigned int computeNumSmallerEigenvals(float             *g_d,
+                                                          float             *g_s,
+                                                          const unsigned int n,
+                                                          const float        x,
+                                                          const unsigned int tid,
+                                                          const unsigned int num_intervals_active,
+                                                          float             *s_d,
+                                                          float             *s_s,
+                                                          unsigned int       converged,
+                                                          cg::thread_block   cta)
+{
+    float        delta = 1.0f;
+    unsigned int count = 0;
+
+    cg::sync(cta);
+
+    // read data into shared memory
+    if (threadIdx.x < n) {
+        s_d[threadIdx.x] = *(g_d + threadIdx.x);
+        s_s[threadIdx.x] = *(g_s + threadIdx.x - 1);
+    }
+
+    cg::sync(cta);
+
+    // perform loop only for active threads
+    if ((tid < num_intervals_active) && (0 == converged)) {
+        // perform (optimized) Gaussian elimination to determine the number
+        // of eigenvalues that are smaller than n
+        for (unsigned int k = 0; k < n; ++k) {
+            delta = s_d[k] - x - (s_s[k] * s_s[k]) / delta;
+            count += (delta < 0) ? 1 : 0;
+        }
+
+    } // end if thread currently processing an interval
+
+    return count;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+//! Compute number of eigenvalues that are smaller than x given a symmetric,
+//! real, and tridiagonal matrix
+//! @param  g_d  diagonal elements stored in global memory
+//! @param  g_s  superdiagonal elements stored in global memory
+//! @param  n    size of matrix
+//! @param  x    value for which the number of eigenvalues that are smaller is
+//!              seeked
+//! @param  tid  thread identified (e.g. threadIdx.x or gtid)
+//! @param  num_intervals_active  number of active intervals / threads that
+//!                               currently process an interval
+//! @param  s_d  scratch space to store diagonal entries of the tridiagonal
+//!              matrix in shared memory
+//! @param  s_s  scratch space to store superdiagonal entries of the tridiagonal
+//!              matrix in shared memory
+//! @param  converged  flag if the current thread is already converged (that
+//!         is count does not have to be computed)
+////////////////////////////////////////////////////////////////////////////////
+__device__ inline unsigned int computeNumSmallerEigenvalsLarge(float             *g_d,
+                                                               float             *g_s,
+                                                               const unsigned int n,
+                                                               const float        x,
+                                                               const unsigned int tid,
+                                                               const unsigned int num_intervals_active,
+                                                               float             *s_d,
+                                                               float             *s_s,
+                                                               unsigned int       converged,
+                                                               cg::thread_block   cta)
+{
+    float        delta = 1.0f;
+    unsigned int count = 0;
+
+    unsigned int rem = n;
+
+    // do until whole diagonal and superdiagonal has been loaded and processed
+    for (unsigned int i = 0; i < n; i += blockDim.x) {
+        cg::sync(cta);
+
+        // read new chunk of data into shared memory
+        if ((i + threadIdx.x) < n) {
+            s_d[threadIdx.x] = *(g_d + i + threadIdx.x);
+            s_s[threadIdx.x] = *(g_s + i + threadIdx.x - 1);
+        }
+
+        cg::sync(cta);
+
+        if (tid < num_intervals_active) {
+            // perform (optimized) Gaussian elimination to determine the number
+            // of eigenvalues that are smaller than n
+            for (unsigned int k = 0; k < min(rem, blockDim.x); ++k) {
+                delta = s_d[k] - x - (s_s[k] * s_s[k]) / delta;
+                // delta = (abs( delta) < (1.0e-10)) ? -(1.0e-10) : delta;
+                count += (delta < 0) ? 1 : 0;
+            }
+
+        } // end if thread currently processing an interval
+
+        rem -= blockDim.x;
+    }
+
+    return count;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+//! Store all non-empty intervals resulting from the subdivision of the interval
+//! currently processed by the thread
+//! @param  addr  base address for storing intervals
+//! @param  num_threads_active  number of threads / intervals in current sweep
+//! @param  s_left  shared memory storage for left interval limits
+//! @param  s_right  shared memory storage for right interval limits
+//! @param  s_left_count  shared memory storage for number of eigenvalues less
+//!                       than left interval limits
+//! @param  s_right_count  shared memory storage for number of eigenvalues less
+//!                       than right interval limits
+//! @param  left   lower limit of interval
+//! @param  mid    midpoint of interval
+//! @param  right  upper limit of interval
+//! @param  left_count  eigenvalues less than \a left
+//! @param  mid_count  eigenvalues less than \a mid
+//! @param  right_count  eigenvalues less than \a right
+//! @param  precision  desired precision for eigenvalues
+//! @param  compact_second_chunk  shared mem flag if second chunk is used and
+//!                               ergo requires compaction
+//! @param  s_compaction_list_exc  helper array for stream compaction,
+//!                                s_compaction_list_exc[tid] = 1 when the
+//!                                thread generated two child intervals
+//! @is_active_interval  mark is thread has a second non-empty child interval
+////////////////////////////////////////////////////////////////////////////////
+template <class S, class T>
+__device__ void storeNonEmptyIntervals(unsigned int       addr,
+                                       const unsigned int num_threads_active,
+                                       float             *s_left,
+                                       float             *s_right,
+                                       T                 *s_left_count,
+                                       T                 *s_right_count,
+                                       float              left,
+                                       float              mid,
+                                       float              right,
+                                       const S            left_count,
+                                       const S            mid_count,
+                                       const S            right_count,
+                                       float              precision,
+                                       unsigned int      &compact_second_chunk,
+                                       T                 *s_compaction_list_exc,
+                                       unsigned int      &is_active_second)
+{
+    // check if both child intervals are valid
+    if ((left_count != mid_count) && (mid_count != right_count)) {
+        // store the left interval
+        storeInterval(addr, s_left, s_right, s_left_count, s_right_count, left, mid, left_count, mid_count, precision);
+
+        // mark that a second interval has been generated, only stored after
+        // stream compaction of second chunk
+        is_active_second                   = 1;
+        s_compaction_list_exc[threadIdx.x] = 1;
+        atomicExch(&compact_second_chunk, 1);
+    }
+    else {
+        // only one non-empty child interval
+
+        // mark that no second child
+        is_active_second                   = 0;
+        s_compaction_list_exc[threadIdx.x] = 0;
+
+        // store the one valid child interval
+        if (left_count != mid_count) {
+            storeInterval(
+                addr, s_left, s_right, s_left_count, s_right_count, left, mid, left_count, mid_count, precision);
+        }
+        else {
+            storeInterval(
+                addr, s_left, s_right, s_left_count, s_right_count, mid, right, mid_count, right_count, precision);
+        }
+    }
+}
+////////////////////////////////////////////////////////////////////////////////
+//! Create indices for compaction, that is process \a s_compaction_list_exc
+//! which is 1 for intervals that generated a second child and 0 otherwise
+//! and create for each of the non-zero elements the index where the new
+//! interval belongs to in a compact representation of all generated second
+//! childs
+//! @param   s_compaction_list_exc  list containing the flags which threads
+//!                                 generated two children
+//! @param   num_threads_compaction number of threads to employ for compaction
+////////////////////////////////////////////////////////////////////////////////
+template <class T>
+__device__ void
+createIndicesCompaction(T *s_compaction_list_exc, unsigned int num_threads_compaction, cg::thread_block cta)
+{
+    unsigned int       offset = 1;
+    const unsigned int tid    = threadIdx.x;
+
+    // higher levels of scan tree
+    for (int d = (num_threads_compaction >> 1); d > 0; d >>= 1) {
+        cg::sync(cta);
+
+        if (tid < d) {
+            unsigned int ai = offset * (2 * tid + 1) - 1;
+            unsigned int bi = offset * (2 * tid + 2) - 1;
+
+            s_compaction_list_exc[bi] = s_compaction_list_exc[bi] + s_compaction_list_exc[ai];
+        }
+
+        offset <<= 1;
+    }
+
+    // traverse down tree: first down to level 2 across
+    for (int d = 2; d < num_threads_compaction; d <<= 1) {
+        offset >>= 1;
+        cg::sync(cta);
+
+        if (tid < (d - 1)) {
+            unsigned int ai = offset * (tid + 1) - 1;
+            unsigned int bi = ai + (offset >> 1);
+
+            s_compaction_list_exc[bi] = s_compaction_list_exc[bi] + s_compaction_list_exc[ai];
+        }
+    }
+
+    cg::sync(cta);
+}
+
+///////////////////////////////////////////////////////////////////////////////
+//! Perform stream compaction for second child intervals
+//! @param  s_left  shared
+//! @param  s_left  shared memory storage for left interval limits
+//! @param  s_right  shared memory storage for right interval limits
+//! @param  s_left_count  shared memory storage for number of eigenvalues less
+//!                       than left interval limits
+//! @param  s_right_count  shared memory storage for number of eigenvalues less
+//!                       than right interval limits
+//! @param  mid    midpoint of current interval (left of new interval)
+//! @param  right  upper limit of interval
+//! @param  mid_count  eigenvalues less than \a mid
+//! @param  s_compaction_list  list containing the indices where the data has
+//!         to be stored
+//! @param  num_threads_active  number of active threads / intervals
+//! @is_active_interval  mark is thread has a second non-empty child interval
+///////////////////////////////////////////////////////////////////////////////
+template <class T>
+__device__ void compactIntervals(float       *s_left,
+                                 float       *s_right,
+                                 T           *s_left_count,
+                                 T           *s_right_count,
+                                 float        mid,
+                                 float        right,
+                                 unsigned int mid_count,
+                                 unsigned int right_count,
+                                 T           *s_compaction_list,
+                                 unsigned int num_threads_active,
+                                 unsigned int is_active_second)
+{
+    const unsigned int tid = threadIdx.x;
+
+    // perform compaction / copy data for all threads where the second
+    // child is not dead
+    if ((tid < num_threads_active) && (1 == is_active_second)) {
+        unsigned int addr_w = num_threads_active + s_compaction_list[tid];
+
+        s_left[addr_w]        = mid;
+        s_right[addr_w]       = right;
+        s_left_count[addr_w]  = mid_count;
+        s_right_count[addr_w] = right_count;
+    }
+}
+
+///////////////////////////////////////////////////////////////////////////////
+//! Store intervals that have already converged (w.r.t. the desired precision),
+//! duplicating intervals that contain multiple eigenvalues
+//! @param  s_left  shared memory storage for left interval limits
+//! @param  s_right  shared memory storage for right interval limits
+//! @param  s_left_count  shared memory storage for number of eigenvalues less
+//!                       than left interval limits
+//! @param  s_right_count  shared memory storage for number of eigenvalues less
+//!                       than right interval limits
+//! @param  left   lower limit of interval
+//! @param  mid    midpoint of interval (updated if split is necessary)
+//! @param  right  upper limit of interval
+//! @param  left_count  eigenvalues less than \a left
+//! @param  mid_count  eigenvalues less than \a mid
+//! @param  right_count  eigenvalues less than \a right
+//! @param  s_compaction_list_exc  helper array for stream compaction, updated
+//!                                at tid if split is necessary
+//! @param  compact_second_chunk  shared mem flag if second chunk is used and
+//!                               ergo requires compaction
+//! @param  num_threads_active  number of active threads / intervals
+///////////////////////////////////////////////////////////////////////////////
+template <class T, class S>
+__device__ void storeIntervalConverged(float             *s_left,
+                                       float             *s_right,
+                                       T                 *s_left_count,
+                                       T                 *s_right_count,
+                                       float             &left,
+                                       float             &mid,
+                                       float             &right,
+                                       S                 &left_count,
+                                       S                 &mid_count,
+                                       S                 &right_count,
+                                       T                 *s_compaction_list_exc,
+                                       unsigned int      &compact_second_chunk,
+                                       const unsigned int num_threads_active)
+{
+    const unsigned int tid          = threadIdx.x;
+    const unsigned int multiplicity = right_count - left_count;
+
+    // check multiplicity of eigenvalue
+    if (1 == multiplicity) {
+        // just re-store intervals, simple eigenvalue
+        s_left[tid]        = left;
+        s_right[tid]       = right;
+        s_left_count[tid]  = left_count;
+        s_right_count[tid] = right_count;
+
+        // mark that no second child / clear
+        s_right_count[tid + num_threads_active] = 0;
+        s_compaction_list_exc[tid]              = 0;
+    }
+    else {
+        // number of eigenvalues after the split less than mid
+        mid_count = left_count + (multiplicity >> 1);
+
+        // store left interval
+        s_left[tid]        = left;
+        s_right[tid]       = right;
+        s_left_count[tid]  = left_count;
+        s_right_count[tid] = mid_count;
+
+        mid = left;
+
+        // mark that second child interval exists
+        s_right_count[tid + num_threads_active] = right_count;
+        s_compaction_list_exc[tid]              = 1;
+        compact_second_chunk                    = 1;
+    }
+}
+
+template <class T, class S>
+__device__ void storeIntervalConverged(float             *s_left,
+                                       float             *s_right,
+                                       T                 *s_left_count,
+                                       T                 *s_right_count,
+                                       float             &left,
+                                       float             &mid,
+                                       float             &right,
+                                       S                 &left_count,
+                                       S                 &mid_count,
+                                       S                 &right_count,
+                                       T                 *s_compaction_list_exc,
+                                       unsigned int      &compact_second_chunk,
+                                       const unsigned int num_threads_active,
+                                       unsigned int      &is_active_second)
+{
+    const unsigned int tid          = threadIdx.x;
+    const unsigned int multiplicity = right_count - left_count;
+
+    // check multiplicity of eigenvalue
+    if (1 == multiplicity) {
+        // just re-store intervals, simple eigenvalue
+        s_left[tid]        = left;
+        s_right[tid]       = right;
+        s_left_count[tid]  = left_count;
+        s_right_count[tid] = right_count;
+
+        // mark that no second child / clear
+        is_active_second           = 0;
+        s_compaction_list_exc[tid] = 0;
+    }
+    else {
+        // number of eigenvalues after the split less than mid
+        mid_count = left_count + (multiplicity >> 1);
+
+        // store left interval
+        s_left[tid]        = left;
+        s_right[tid]       = right;
+        s_left_count[tid]  = left_count;
+        s_right_count[tid] = mid_count;
+
+        mid = left;
+
+        // mark that second child interval exists
+        is_active_second           = 1;
+        s_compaction_list_exc[tid] = 1;
+        compact_second_chunk       = 1;
+    }
+}
+
+///////////////////////////////////////////////////////////////////////////////
+//! Subdivide interval if active and not already converged
+//! @param tid  id of thread
+//! @param  s_left  shared memory storage for left interval limits
+//! @param  s_right  shared memory storage for right interval limits
+//! @param  s_left_count  shared memory storage for number of eigenvalues less
+//!                       than left interval limits
+//! @param  s_right_count  shared memory storage for number of eigenvalues less
+//!                       than right interval limits
+//! @param  num_threads_active  number of active threads in warp
+//! @param  left   lower limit of interval
+//! @param  right  upper limit of interval
+//! @param  left_count  eigenvalues less than \a left
+//! @param  right_count  eigenvalues less than \a right
+//! @param  all_threads_converged  shared memory flag if all threads are
+//!                                 converged
+///////////////////////////////////////////////////////////////////////////////
+template <class T>
+__device__ void subdivideActiveInterval(const unsigned int tid,
+                                        float             *s_left,
+                                        float             *s_right,
+                                        T                 *s_left_count,
+                                        T                 *s_right_count,
+                                        const unsigned int num_threads_active,
+                                        float             &left,
+                                        float             &right,
+                                        unsigned int      &left_count,
+                                        unsigned int      &right_count,
+                                        float             &mid,
+                                        unsigned int      &all_threads_converged)
+{
+    // for all active threads
+    if (tid < num_threads_active) {
+        left        = s_left[tid];
+        right       = s_right[tid];
+        left_count  = s_left_count[tid];
+        right_count = s_right_count[tid];
+
+        // check if thread already converged
+        if (left != right) {
+            mid = computeMidpoint(left, right);
+            atomicExch(&all_threads_converged, 0);
+        }
+        else if ((right_count - left_count) > 1) {
+            // mark as not converged if multiple eigenvalues enclosed
+            // duplicate interval in storeIntervalsConverged()
+            atomicExch(&all_threads_converged, 0);
+        }
+
+    } // end for all active threads
+}
+
+#endif // #ifndef _BISECT_UTIL_H_

cpp/2_Concepts_and_Techniques/eigenvalues/config.h

@@ -0,0 +1,41 @@
+/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/* Global configuration parameter */
+
+#ifndef _CONFIG_H_
+#define _CONFIG_H_
+
+// should be power of two
+#define MAX_THREADS_BLOCK 256
+
+#define MAX_SMALL_MATRIX               512
+#define MAX_THREADS_BLOCK_SMALL_MATRIX 512
+
+#define MIN_ABS_INTERVAL 5.0e-37
+
+#endif // #ifndef _CONFIG_H_

cpp/2_Concepts_and_Techniques/eigenvalues/gerschgorin.cpp

@@ -0,0 +1,84 @@
+/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/* Computation of Gerschgorin interval for symmetric, tridiagonal matrix */
+
+
+#include "gerschgorin.h"
+
+#include <cfloat>
+#include <cmath>
+#include <cstdio>
+#include <cstdlib>
+
+#include "util.h"
+
+////////////////////////////////////////////////////////////////////////////////
+//! Compute Gerschgorin interval for symmetric, tridiagonal matrix
+//! @param  d  diagonal elements
+//! @param  s  superdiagonal elements
+//! @param  n  size of matrix
+//! @param  lg  lower limit of Gerschgorin interval
+//! @param  ug  upper limit of Gerschgorin interval
+////////////////////////////////////////////////////////////////////////////////
+void computeGerschgorin(float *d, float *s, unsigned int n, float &lg, float &ug)
+{
+    lg = FLT_MAX;
+    ug = -FLT_MAX;
+
+    // compute bounds
+    for (unsigned int i = 1; i < (n - 1); ++i) {
+        // sum over the absolute values of all elements of row i
+        float sum_abs_ni = fabsf(s[i - 1]) + fabsf(s[i]);
+
+        lg = min(lg, d[i] - sum_abs_ni);
+        ug = max(ug, d[i] + sum_abs_ni);
+    }
+
+    // first and last row, only one superdiagonal element
+
+    // first row
+    lg = min(lg, d[0] - fabsf(s[0]));
+    ug = max(ug, d[0] + fabsf(s[0]));
+
+    // last row
+    lg = min(lg, d[n - 1] - fabsf(s[n - 2]));
+    ug = max(ug, d[n - 1] + fabsf(s[n - 2]));
+
+    // increase interval to avoid side effects of fp arithmetic
+    float bnorm = max(fabsf(ug), fabsf(lg));
+
+    // these values depend on the implementation of floating count that is
+    // employed in the following
+    float psi_0 = 11 * FLT_EPSILON * bnorm;
+    float psi_n = 11 * FLT_EPSILON * bnorm;
+
+    lg = lg - bnorm * 2 * n * FLT_EPSILON - psi_0;
+    ug = ug + bnorm * 2 * n * FLT_EPSILON + psi_n;
+
+    ug = max(lg, ug);
+}

cpp/2_Concepts_and_Techniques/eigenvalues/gerschgorin.h

@@ -0,0 +1,43 @@
+/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/* Computation of Gerschgorin interval for symmetric, tridiagonal matrix */
+
+#ifndef _GERSCHGORIN_H_
+#define _GERSCHGORIN_H_
+
+////////////////////////////////////////////////////////////////////////////////
+//! Compute Gerschgorin interval for symmetric, tridiagonal matrix
+//! @param  d  diagonal elements
+//! @param  s  superdiagonal elements
+//! @param  n  size of matrix
+//! @param  lg  lower limit of Gerschgorin interval
+//! @param  ug  upper limit of Gerschgorin interval
+////////////////////////////////////////////////////////////////////////////////
+extern "C" void computeGerschgorin(float *d, float *s, unsigned int n, float &lg, float &ug);
+
+#endif // #ifndef _GERSCHGORIN_H_

cpp/2_Concepts_and_Techniques/eigenvalues/main.cu

@@ -0,0 +1,325 @@
+/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/* Computation of eigenvalues of symmetric, tridiagonal matrix using
+ * bisection.
+ */
+
+// includes, system
+#include <assert.h>
+#include <float.h>
+#include <math.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+// includes, project
+#include <helper_cuda.h>
+#include <helper_functions.h>
+
+#include "bisect_large.cuh"
+#include "bisect_small.cuh"
+#include "config.h"
+#include "gerschgorin.h"
+#include "matlab.h"
+#include "structs.h"
+#include "util.h"
+
+////////////////////////////////////////////////////////////////////////////////
+// declaration, forward
+bool runTest(int argc, char **argv);
+
+////////////////////////////////////////////////////////////////////////////////
+// Program main
+////////////////////////////////////////////////////////////////////////////////
+int main(int argc, char **argv)
+{
+    bool bQAResults = false;
+
+    printf("Starting eigenvalues\n");
+
+    bQAResults = runTest(argc, argv);
+    printf("Test %s\n", bQAResults ? "Succeeded!" : "Failed!");
+
+    exit(bQAResults ? EXIT_SUCCESS : EXIT_FAILURE);
+}
+
+////////////////////////////////////////////////////////////////////////////////
+//! Initialize the input data to the algorithm
+//! @param input  handles to the input data
+//! @param exec_path  path where executable is run (argv[0])
+//! @param mat_size  size of the matrix
+//! @param user_defined  1 if the matrix size has been requested by the user,
+//!                      0 if the default size
+////////////////////////////////////////////////////////////////////////////////
+void initInputData(InputData &input, char *exec_path, const unsigned int mat_size, const unsigned int user_defined)
+{
+    // allocate memory
+    input.a = (float *)malloc(sizeof(float) * mat_size);
+    input.b = (float *)malloc(sizeof(float) * mat_size);
+
+    if (1 == user_defined) {
+        // initialize diagonal and superdiagonal entries with random values
+        srand(278217421);
+
+        // srand( clock());
+        for (unsigned int i = 0; i < mat_size; ++i) {
+            input.a[i] = (float)(2.0 * (((double)rand() / (double)RAND_MAX) - 0.5));
+            input.b[i] = (float)(2.0 * (((double)rand() / (double)RAND_MAX) - 0.5));
+        }
+
+        // the first element of s is used as padding on the device (thus the
+        // whole vector is copied to the device but the kernels are launched
+        // with (s+1) as start address
+        input.b[0] = 0.0f;
+    }
+    else {
+        // read default matrix
+        unsigned int input_data_size = mat_size;
+        char        *diag_path       = sdkFindFilePath("diagonal.dat", exec_path);
+        assert(NULL != diag_path);
+        sdkReadFile(diag_path, &(input.a), &input_data_size, false);
+
+        char *sdiag_path = sdkFindFilePath("superdiagonal.dat", exec_path);
+        assert(NULL != sdiag_path);
+        sdkReadFile(sdiag_path, &(input.b), &input_data_size, false);
+
+        free(diag_path);
+        free(sdiag_path);
+    }
+
+    // allocate device memory for input
+    checkCudaErrors(cudaMalloc((void **)&(input.g_a), sizeof(float) * mat_size));
+    checkCudaErrors(cudaMalloc((void **)&(input.g_b_raw), sizeof(float) * mat_size));
+
+    // copy data to device
+    checkCudaErrors(cudaMemcpy(input.g_a, input.a, sizeof(float) * mat_size, cudaMemcpyHostToDevice));
+    checkCudaErrors(cudaMemcpy(input.g_b_raw, input.b, sizeof(float) * mat_size, cudaMemcpyHostToDevice));
+
+    input.g_b = input.g_b_raw + 1;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+//! Clean up input data, in particular allocated memory
+//! @param input  handles to the input data
+////////////////////////////////////////////////////////////////////////////////
+void cleanupInputData(InputData &input)
+{
+    freePtr(input.a);
+    freePtr(input.b);
+
+    checkCudaErrors(cudaFree(input.g_a));
+    input.g_a = NULL;
+    checkCudaErrors(cudaFree(input.g_b_raw));
+    input.g_b_raw = NULL;
+    input.g_b     = NULL;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+//! Check if a specific matrix size has to be used
+//! @param argc  number of command line arguments (from main(argc, argv)
+//! @param argv  pointers to command line arguments (from main(argc, argv)
+//! @param matrix_size  size of matrix, updated if specific size specified on
+//!                     command line
+////////////////////////////////////////////////////////////////////////////////
+void getMatrixSize(int argc, char **argv, unsigned int &mat_size, unsigned int &user_defined)
+{
+    int temp = -1;
+
+    if (checkCmdLineFlag(argc, (const char **)argv, "matrix-size")) {
+        temp = getCmdLineArgumentInt(argc, (const char **)argv, "matrix-size");
+    }
+
+    if (temp > 0) {
+        mat_size = (unsigned int)temp;
+        // data type short is used in the kernel
+        assert(mat_size < (1 << 16));
+
+        // mat_size should be large than 2
+        assert(mat_size >= 2);
+
+        user_defined = 1;
+    }
+
+    printf("Matrix size: %i x %i\n", mat_size, mat_size);
+}
+
+////////////////////////////////////////////////////////////////////////////////
+//! Check if a specific precision of the eigenvalue has to be obtained
+//! @param argc  number of command line arguments (from main(argc, argv)
+//! @param argv  pointers to command line arguments (from main(argc, argv)
+//! @param iters_timing  numbers of iterations for timing, updated if a
+//!                      specific number is specified on the command line
+//! @param user_defined  1 if the precision has been requested by the user,
+//!                      0 if the default size
+////////////////////////////////////////////////////////////////////////////////
+void getPrecision(int argc, char **argv, float &precision, unsigned int &user_defined)
+{
+    float temp = -1.0f;
+
+    if (checkCmdLineFlag(argc, (const char **)argv, "precision")) {
+        temp = getCmdLineArgumentFloat(argc, (const char **)argv, "precision");
+        printf("Precision is between [0.001, 0.000001]\n");
+    }
+
+    if (temp > 1e-6 && temp <= 0.001) {
+        precision    = temp;
+        user_defined = 1;
+    }
+
+    printf("Precision: %f\n", precision);
+}
+
+////////////////////////////////////////////////////////////////////////////////
+//! Check if a particular number of iterations for timings has to be used
+//! @param argc  number of command line arguments (from main(argc, argv)
+//! @param argv  pointers to command line arguments (from main(argc, argv)
+//! @param  iters_timing  number of timing iterations, updated if user
+//!                       specific value
+////////////////////////////////////////////////////////////////////////////////
+void getItersTiming(int argc, char **argv, unsigned int &iters_timing)
+{
+    int temp = -1;
+
+    if (checkCmdLineFlag(argc, (const char **)argv, "iters-timing")) {
+        temp = getCmdLineArgumentInt(argc, (const char **)argv, "iters-timing");
+    }
+
+    if (temp > 0) {
+        iters_timing = temp;
+    }
+
+    printf("Iterations to be timed: %i\n", iters_timing);
+}
+
+////////////////////////////////////////////////////////////////////////////////
+//! Check if a particular filename has to be used for the file where the result
+//! is stored
+//! @param argc  number of command line arguments (from main(argc, argv)
+//! @param argv  pointers to command line arguments (from main(argc, argv)
+//! @param  filename  filename of result file, updated if user specified
+//!                   filename
+////////////////////////////////////////////////////////////////////////////////
+void getResultFilename(int argc, char **argv, char *&filename)
+{
+    char *temp = NULL;
+    getCmdLineArgumentString(argc, (const char **)argv, "filename-result", &temp);
+
+    if (NULL != temp) {
+        filename = (char *)malloc(sizeof(char) * strlen(temp));
+        strcpy(filename, temp);
+
+        free(temp);
+    }
+
+    printf("Result filename: '%s'\n", filename);
+}
+
+////////////////////////////////////////////////////////////////////////////////
+//! Run a simple test for CUDA
+////////////////////////////////////////////////////////////////////////////////
+bool runTest(int argc, char **argv)
+{
+    bool bCompareResult = false;
+
+    findCudaDevice(argc, (const char **)argv);
+
+    StopWatchInterface *timer       = NULL;
+    StopWatchInterface *timer_total = NULL;
+    sdkCreateTimer(&timer);
+    sdkCreateTimer(&timer_total);
+
+    // default
+    unsigned int mat_size = 2048;
+    // flag if the matrix size is due to explicit user request
+    unsigned int user_defined = 0;
+    // desired precision of eigenvalues
+    float        precision    = 0.00001f;
+    unsigned int iters_timing = 100;
+    char        *result_file  = (char *)"eigenvalues.dat";
+
+    // check if there is a command line request for the matrix size
+    getMatrixSize(argc, argv, mat_size, user_defined);
+
+    // check if user requested specific precision
+    getPrecision(argc, argv, precision, user_defined);
+
+    // check if user requested specific number of iterations for timing
+    getItersTiming(argc, argv, iters_timing);
+
+    // file name for result file
+    getResultFilename(argc, argv, result_file);
+
+    // set up input
+    InputData input;
+    initInputData(input, argv[0], mat_size, user_defined);
+
+    // compute Gerschgorin interval
+    float lg = FLT_MAX;
+    float ug = -FLT_MAX;
+    computeGerschgorin(input.a, input.b + 1, mat_size, lg, ug);
+    printf("Gerschgorin interval: %f / %f\n", lg, ug);
+
+    // two kernels, for small matrices a lot of overhead can be avoided
+    if (mat_size <= MAX_SMALL_MATRIX) {
+        // initialize memory for result
+        ResultDataSmall result;
+        initResultSmallMatrix(result, mat_size);
+
+        // run the kernel
+        computeEigenvaluesSmallMatrix(input, result, mat_size, lg, ug, precision, iters_timing);
+
+        // get the result from the device and do some sanity checks,
+        // save the result
+        processResultSmallMatrix(input, result, mat_size, result_file);
+
+        // clean up
+        cleanupResultSmallMatrix(result);
+
+        printf("User requests non-default argument(s), skipping self-check!\n");
+        bCompareResult = true;
+    }
+    else {
+        // initialize memory for result
+        ResultDataLarge result;
+        initResultDataLargeMatrix(result, mat_size);
+
+        // run the kernel
+        computeEigenvaluesLargeMatrix(input, result, mat_size, precision, lg, ug, iters_timing);
+
+        // get the result from the device and do some sanity checks
+        // save the result if user specified matrix size
+        bCompareResult = processResultDataLargeMatrix(input, result, mat_size, result_file, user_defined, argv[0]);
+
+        // cleanup
+        cleanupResultDataLargeMatrix(result);
+    }
+
+    cleanupInputData(input);
+
+    return bCompareResult;
+}

cpp/2_Concepts_and_Techniques/eigenvalues/matlab.cpp

@@ -0,0 +1,70 @@
+/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+//! includes, system
+#include <algorithm>
+#include <cassert>
+#include <cmath>
+#include <fstream>
+#include <iomanip>
+#include <iostream>
+#include <map>
+#include <vector>
+
+// includes, projcet
+#include "matlab.h"
+
+// namespace, unnamed
+namespace {
+} // namespace
+
+///////////////////////////////////////////////////////////////////////////////
+//! Write a tridiagonal, symmetric matrix in vector representation and
+//! it's eigenvalues
+//! @param  filename  name of output file
+//! @param  d  diagonal entries of the matrix
+//! @param  s  superdiagonal entries of the matrix (len = n - 1)
+//! @param  eigenvals  eigenvalues of the matrix
+//! @param  indices  vector of len n containing the position of the eigenvalues
+//!                  if these are sorted in ascending order
+//! @param  n  size of the matrix
+///////////////////////////////////////////////////////////////////////////////
+void writeTridiagSymMatlab(const char *filename, float *d, float *s, float *eigenvals, const unsigned int n)
+{
+    std::ofstream file(filename, std::ios::out);
+
+    // write diagonal entries
+    writeVectorMatlab(file, "d", d, n);
+
+    // write superdiagonal entries
+    writeVectorMatlab(file, "s", s, n - 1);
+
+    // write eigenvalues
+    writeVectorMatlab(file, "eigvals", eigenvals, n);
+
+    file.close();
+}

cpp/2_Concepts_and_Techniques/eigenvalues/matlab.h

@@ -0,0 +1,120 @@
+/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/* Header for utility functionality.
+ * Host code.
+ */
+
+#ifndef _MATLAB_H_
+#define _MATLAB_H_
+
+// includes, system
+#include <math.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+// includes, project
+
+////////////////////////////////////////////////////////////////////////////////
+//! Write a tridiagonal, symmetric matrix in vector representation and
+//! it's eigenvalues
+//! @param  filename  name of output file
+//! @param  d  diagonal entries of the matrix
+//! @param  s  superdiagonal entries of the matrix (len = n - 1)
+//! @param  eigenvals  eigenvalues of the matrix
+//! @param  indices  vector of len n containing the position of the eigenvalues
+//!                  if these are sorted in ascending order
+//! @param  n  size of the matrix
+////////////////////////////////////////////////////////////////////////////////
+extern "C" void writeTridiagSymMatlab(const char *filename, float *d, float *s, float *eigenvals, const unsigned int n);
+
+////////////////////////////////////////////////////////////////////////////////
+//! Write matrix to a file in Matlab format
+//! @param  file  file handle to which to write he matrix
+//! @param  mat_name  name of matrix in Matlab
+//! @param  mat  matrix to write to the file
+//! @param  mat_size  size of the (square) matrix \a mat
+////////////////////////////////////////////////////////////////////////////////
+template <class T, class S> void writeMatrixMatlab(T &file, const char *mat_name, S *&mat, const unsigned int mat_size);
+
+////////////////////////////////////////////////////////////////////////////////
+//! Write vector to a file in Matlab format
+//! @param  file  file handle to which to write he matrix
+//! @param  vec_name  name of vector in Matlab
+//! @param  vec  matrix to write to the file
+//! @param  vec_len  length of the vector
+////////////////////////////////////////////////////////////////////////////////
+template <class T, class S> void writeVectorMatlab(T &file, const char *vec_name, S *&vec, const unsigned int vec_len);
+
+// implementations
+
+////////////////////////////////////////////////////////////////////////////////
+//! Write matrix to a file in Matlab format
+//! @param  file  file handle to which to write he matrix
+//! @param  mat_name  name of matrix in Matlab
+//! @param  mat  matrix to write to the file
+//! @param  mat_size  size of the (square) matrix \a mat
+////////////////////////////////////////////////////////////////////////////////
+template <class T, class S> void writeMatrixMatlab(T &file, const char *mat_name, S *&mat, const unsigned int mat_size)
+{
+    const unsigned int pitch = sizeof(S) * mat_size;
+
+    file << mat_name << " = [";
+
+    for (unsigned int i = 0; i < mat_size; ++i) {
+        for (unsigned int j = 0; j < mat_size; ++j) {
+            file << getMatrix(mat, pitch, i, j) << " ";
+        }
+
+        if (i != mat_size - 1) {
+            file << "; ";
+        }
+    }
+
+    file << "];\n";
+}
+
+////////////////////////////////////////////////////////////////////////////////
+//! Write vector to a file in Matlab format
+//! @param  file  file handle to which to write he matrix
+//! @param  vec_name  name of vector in Matlab
+//! @param  vec  matrix to write to the file
+//! @param  vec_len  length of the vector
+////////////////////////////////////////////////////////////////////////////////
+template <class T, class S> void writeVectorMatlab(T &file, const char *vec_name, S *&vec, const unsigned int vec_len)
+{
+    file << vec_name << " = [";
+
+    for (unsigned int i = 0; i < vec_len; ++i) {
+        file << vec[i] << " ";
+    }
+
+    file << "];\n";
+}
+
+#endif // _MATLAB_H_

cpp/2_Concepts_and_Techniques/eigenvalues/structs.h

@@ -0,0 +1,132 @@
+/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/* Helper structures to simplify variable handling */
+
+#ifndef _STRUCTS_H_
+#define _STRUCTS_H_
+
+struct InputData
+{
+    //! host side representation of diagonal
+    float *a;
+    //! host side representation superdiagonal
+    float *b;
+
+    //! device side representation of diagonal
+    float *g_a;
+    //! device side representation of superdiagonal
+    float *g_b;
+    //! helper variable pointing to the mem allocated for g_b which provides
+    //! space for one additional element of padding at the beginning
+    float *g_b_raw;
+};
+
+struct ResultDataSmall
+{
+    //! eigenvalues (host side)
+    float *eigenvalues;
+
+    // left interval limits at the end of the computation
+    float *g_left;
+
+    // right interval limits at the end of the computation
+    float *g_right;
+
+    // number of eigenvalues smaller than the left interval limit
+    unsigned int *g_left_count;
+
+    // number of eigenvalues bigger than the right interval limit
+    unsigned int *g_right_count;
+
+    //! flag if algorithm converged
+    unsigned int *g_converged;
+
+    // helper variables
+
+    unsigned int mat_size_f;
+    unsigned int mat_size_ui;
+
+    float        *zero_f;
+    unsigned int *zero_ui;
+};
+
+struct ResultDataLarge
+{
+    // number of intervals containing one eigenvalue after the first step
+    unsigned int *g_num_one;
+
+    // number of (thread) blocks of intervals containing multiple eigenvalues
+    // after the first step
+    unsigned int *g_num_blocks_mult;
+
+    //! left interval limits of intervals containing one eigenvalue after the
+    //! first iteration step
+    float *g_left_one;
+
+    //! right interval limits of intervals containing one eigenvalue after the
+    //! first iteration step
+    float *g_right_one;
+
+    //! interval indices (position in sorted listed of eigenvalues)
+    //! of intervals containing one eigenvalue after the first iteration step
+    unsigned int *g_pos_one;
+
+    //! left interval limits of intervals containing multiple eigenvalues
+    //! after the first iteration step
+    float *g_left_mult;
+
+    //! right interval limits of intervals containing multiple eigenvalues
+    //! after the first iteration step
+    float *g_right_mult;
+
+    //! number of eigenvalues less than the left limit of the eigenvalue
+    //! intervals containing multiple eigenvalues
+    unsigned int *g_left_count_mult;
+
+    //! number of eigenvalues less than the right limit of the eigenvalue
+    //! intervals containing multiple eigenvalues
+    unsigned int *g_right_count_mult;
+
+    //! start addresses in g_left_mult etc. of blocks of intervals containing
+    //! more than one eigenvalue after the first step
+    unsigned int *g_blocks_mult;
+
+    //! accumulated number of intervals in g_left_mult etc. of blocks of
+    //! intervals containing more than one eigenvalue after the first step
+    unsigned int *g_blocks_mult_sum;
+
+    //! eigenvalues that have been generated in the second step from intervals
+    //! that still contained multiple eigenvalues after the first step
+    float *g_lambda_mult;
+
+    //! eigenvalue index of intervals that have been generated in the second
+    //! processing step
+    unsigned int *g_pos_mult;
+};
+
+#endif // #ifndef _STRUCTS_H_

cpp/2_Concepts_and_Techniques/eigenvalues/util.h

@@ -0,0 +1,138 @@
+/* Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of NVIDIA CORPORATION nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/* Utility functions */
+
+#ifndef _UTIL_H_
+#define _UTIL_H_
+
+////////////////////////////////////////////////////////////////////////////////
+//! Safely free() for pointer
+////////////////////////////////////////////////////////////////////////////////
+template <class T> inline void freePtr(T *&ptr)
+{
+    if (NULL != ptr) {
+        free(ptr);
+        ptr = NULL;
+    }
+}
+
+////////////////////////////////////////////////////////////////////////////////
+//! Minimum
+////////////////////////////////////////////////////////////////////////////////
+template <class T>
+#ifdef __CUDACC__
+__host__
+    __device__
+#endif
+        T
+        min(const T &lhs, const T &rhs)
+{
+
+    return (lhs < rhs) ? lhs : rhs;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+//! Maximum
+////////////////////////////////////////////////////////////////////////////////
+template <class T>
+#ifdef __CUDACC__
+__host__
+    __device__
+#endif
+        T
+        max(const T &lhs, const T &rhs)
+{
+
+    return (lhs < rhs) ? rhs : lhs;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+//! Sign of number (integer data type)
+////////////////////////////////////////////////////////////////////////////////
+template <class T>
+#ifdef __CUDACC__
+__host__
+    __device__
+#endif
+        T
+        sign_i(const T &val)
+{
+    return (val < 0) ? -1 : 1;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+//! Sign of number (float)
+////////////////////////////////////////////////////////////////////////////////
+#ifdef __CUDACC__
+__host__ __device__
+#endif
+    inline float
+    sign_f(const float &val)
+{
+    return (val < 0.0f) ? -1.0f : 1.0f;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+//! Sign of number (double)
+////////////////////////////////////////////////////////////////////////////////
+#ifdef __CUDACC__
+__host__ __device__
+#endif
+    inline double
+    sign_d(const double &val)
+{
+    return (val < 0.0) ? -1.0 : 1.0;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+//! Swap \a lhs and \a rhs
+////////////////////////////////////////////////////////////////////////////////
+template <class T>
+#ifdef __CUDACC__
+__host__ __device__
+#endif
+    void
+    swap(T &lhs, T &rhs)
+{
+
+    T temp = rhs;
+    rhs    = lhs;
+    lhs    = temp;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+//! Get the number of blocks that are required to process \a num_threads with
+//! \a num_threads_blocks threads per block
+///////////////////////////////////////////////////////////////////////////////
+extern "C" inline unsigned int getNumBlocksLinear(const unsigned int num_threads, const unsigned int num_threads_block)
+{
+    const unsigned int block_rem = ((num_threads % num_threads_block) != 0) ? 1 : 0;
+    return (num_threads / num_threads_block) + block_rem;
+}
+
+#endif // #ifndef _UTIL_H_