From c8034f368a411b23c4607242acfda2430a4cdcb9 Mon Sep 17 00:00:00 2001 From: Rob Armstrong Date: Fri, 28 Mar 2025 15:07:07 -0700 Subject: [PATCH] Add helper utility to test run all built samples (see README.md for usage details) --- .gitignore | 3 + README.md | 152 +++++++++++++++++++++++ run_tests.py | 215 ++++++++++++++++++++++++++++++++ test_args.json | 329 +++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 699 insertions(+) create mode 100644 run_tests.py create mode 100644 test_args.json diff --git a/.gitignore b/.gitignore index 2c315a33..4b782397 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ build .vs .clangd +test +settings.json +launch.json diff --git a/README.md b/README.md index a708d373..dfc15946 100644 --- a/README.md +++ b/README.md @@ -139,6 +139,158 @@ Note that in the current branch sample cross-compilation for QNX is not fully va near future with QNX cross-compilation instructions. In the meantime, if you want to cross-compile for QNX please check out one of the previous tags prior to the CMake build system transition in 12.8. +## Running All Samples as Tests + +It's important to note that the CUDA samples are _not_ intended as a validation suite for CUDA. They do not cover corner cases, they do not completely cover the +runtime and driver APIs, etc. That said, it can sometimes be useful to run all of the samples as a quick sanity check and we provide a script to do so, `run_tests.py`. + +This Python3 script finds all executables in a subdirectory you choose, matching application names with command line arguments specified in `test_args.json`. It accepts +the following command line arguments: + +| Switch | Purpose | Example | +| -------- | -------------------------------------------------------------------------------------------------------------- | ----------------------- | +| --dir | Specify the root directory to search for executables (recursively) | --dir ./build/Samples | +| --config | JSON configuration file for executable arguments | --config test_args.json | +| --output | Output directory for test results (stdout saved to .txt files - directory will be created if it doesn't exist) | --output ./test | +| --args | Global arguments to pass to all executables (not currently used) | --args arg_1 arg_2 ... | + +Application configurations are loaded from `test_args.json` and matched against executable names (discarding the `.exe` extension on Windows). + +The script returns 0 on success, or the first non-zero error code encountered during testing on failure. It will also print a condensed list of samples that failed, if any. + +There are three primary modes of configuration: + +**Skip** + +An executable configured with "skip" will not be executed. These generally rely on having attached graphical displays and are not suited to this kind of automation. + +Configuration example: +```json +"fluidsGL": { + "skip": true +} +``` + +You will see: +``` +Skipping fluidsGL (marked as skip in config) +``` + +**Single Run** + +For executables to run one time only with arguments, specify each argument as a list entry. Each entry in the JSON file will be appended to the command line, separated +by a space. + +Configuration example: +```json +"ptxgen": { + "args": [ + "test.ll", + "-arch=compute_75" + ] +} +``` + +You will see: +``` +Running ptxgen + Command: ./ptxgen test.ll -arch=compute_75 + Test completed with return code 0 +``` + +**Multiple Runs** + +For executables to run multiple times with different command line arguments, specify any number of sets of args within a "runs" list. + +Configuration example: +```json +"recursiveGaussian": { + "runs": [ + { + "args": [ + "-sigma=10", + "-file=data/ref_10.ppm" + ] + }, + { + "args": [ + "-sigma=14", + "-file=data/ref_14.ppm" + ] + }, + { + "args": [ + "-sigma=18", + "-file=data/ref_18.ppm" + ] + }, + { + "args": [ + "-sigma=22", + "-file=data/ref_22.ppm" + ] + } + ] +} +``` + +You will see: +``` +Running recursiveGaussian (run 1/4) + Command: ./recursiveGaussian -sigma=10 -file=data/ref_10.ppm + Test completed with return code 0 +Running recursiveGaussian (run 2/4) + Command: ./recursiveGaussian -sigma=14 -file=data/ref_14.ppm + Test completed with return code 0 +Running recursiveGaussian (run 3/4) + Command: ./recursiveGaussian -sigma=18 -file=data/ref_18.ppm + Test completed with return code 0 +Running recursiveGaussian (run 4/4) + Command: ./recursiveGaussian -sigma=22 -file=data/ref_22.ppm + Test completed with return code 0 +``` + +### Example Usage + +Here is an example set of commands to build and test all of the samples. + +First, build: +```bash +mkdir build +cd build +cmake .. +make -j$(nproc) +``` + +Now, return to the samples root directory and run the test script: +```bash +cd .. +python3 run_tests.py --output ./test --dir ./build/Samples --config test_args.json +``` + +If all applications run successfully, you will see something similar to this (the specific number of samples will depend on your build type +and system configuration): + +``` +Test Summary: +Ran 181 tests +All tests passed! +``` + +If some samples fail, you will see something like this: + +``` +Test Summary: +Ran 181 tests +Failed tests (2): + volumeFiltering: returned 1 + postProcessGL: returned 1 +``` + +You can inspect the stdout logs in the output directory (generally `APM_.txt` or `APM_.run.txt`) to help +determine what may have gone wrong from the output logs. Please file issues against the samples repository if you believe a sample is failing +incorrectly on your system. + ## Samples list ### [0. Introduction](./Samples/0_Introduction/README.md) diff --git a/run_tests.py b/run_tests.py new file mode 100644 index 00000000..50320c81 --- /dev/null +++ b/run_tests.py @@ -0,0 +1,215 @@ +## Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +## +## Redistribution and use in source and binary forms, with or without +## modification, are permitted provided that the following conditions +## are met: +## * Redistributions of source code must retain the above copyright +## notice, this list of conditions and the following disclaimer. +## * Redistributions in binary form must reproduce the above copyright +## notice, this list of conditions and the following disclaimer in the +## documentation and/or other materials provided with the distribution. +## * Neither the name of NVIDIA CORPORATION nor the names of its +## contributors may be used to endorse or promote products derived +## from this software without specific prior written permission. +## +## THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +## EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +## IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +## PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +## CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +## EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +## PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +## PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +## OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +## (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +## OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +## +## For additional information on the license terms, see the CUDA EULA at +## https://docs.nvidia.com/cuda/eula/index.html + +import os +import sys +import json +import subprocess +import argparse +from pathlib import Path + +def normalize_exe_name(name): + """Normalize executable name across platforms by removing .exe if present""" + return Path(name).stem + +def load_args_config(config_file): + """Load arguments configuration from JSON file""" + if not config_file or not os.path.exists(config_file): + return {} + + try: + with open(config_file, 'r') as f: + config = json.load(f) + + # Validate the config format + if not isinstance(config, dict): + print("Warning: Config file must contain a dictionary/object") + return {} + + return config + except json.JSONDecodeError: + print("Warning: Failed to parse config file as JSON") + return {} + except Exception as e: + print(f"Warning: Error reading config file: {str(e)}") + return {} + +def find_executables(root_dir): + """Find all executable files recursively""" + executables = [] + + for path in Path(root_dir).rglob('*'): + # Skip directories + if not path.is_file(): + continue + + # Check if file is executable + if os.access(path, os.X_OK): + # Skip if it's a library file + if path.suffix.lower() in ('.dll', '.so', '.dylib'): + continue + executables.append(path) + + return executables + +def run_test(executable, output_dir, args_config, global_args=None): + """Run a single test and capture output""" + exe_path = str(executable) + exe_name = executable.name + base_name = normalize_exe_name(exe_name) + + # Check if this executable should be skipped + if base_name in args_config and args_config[base_name].get("skip", False): + print(f"Skipping {exe_name} (marked as skip in config)") + return 0 + + # Get argument sets for this executable + arg_sets = [] + if base_name in args_config: + config = args_config[base_name] + if "args" in config: + # Single argument set (backwards compatibility) + if isinstance(config["args"], list): + arg_sets.append(config["args"]) + else: + print(f"Warning: Arguments for {base_name} must be a list") + elif "runs" in config: + # Multiple argument sets + for run in config["runs"]: + if isinstance(run.get("args", []), list): + arg_sets.append(run.get("args", [])) + else: + print(f"Warning: Arguments for {base_name} run must be a list") + + # If no specific args defined, run once with no args + if not arg_sets: + arg_sets.append([]) + + # Run for each argument set + failed = False + run_number = 1 + for args in arg_sets: + # Create output file name with run number if multiple runs + if len(arg_sets) > 1: + output_file = os.path.abspath(f"{output_dir}/APM_{exe_name}.run{run_number}.txt") + print(f"Running {exe_name} (run {run_number}/{len(arg_sets)})") + else: + output_file = os.path.abspath(f"{output_dir}/APM_{exe_name}.txt") + print(f"Running {exe_name}") + + try: + # Prepare command with arguments + cmd = [f"./{exe_name}"] + cmd.extend(args) + + # Add global arguments if provided + if global_args: + cmd.extend(global_args) + + print(f" Command: {' '.join(cmd)}") + + # Store current directory + original_dir = os.getcwd() + + try: + # Change to executable's directory + os.chdir(os.path.dirname(exe_path)) + + # Run the executable and capture output + with open(output_file, 'w') as f: + result = subprocess.run( + cmd, + stdout=f, + stderr=subprocess.STDOUT, + timeout=300 # 5 minute timeout + ) + + if result.returncode != 0: + failed = True + print(f" Test completed with return code {result.returncode}") + + finally: + # Always restore original directory + os.chdir(original_dir) + + except subprocess.TimeoutExpired: + print(f"Error: {exe_name} timed out after 5 minutes") + failed = True + except Exception as e: + print(f"Error running {exe_name}: {str(e)}") + failed = True + + run_number += 1 + + return 1 if failed else 0 + +def main(): + parser = argparse.ArgumentParser(description='Run all executables and capture output') + parser.add_argument('--dir', default='.', help='Root directory to search for executables') + parser.add_argument('--config', help='JSON configuration file for executable arguments') + parser.add_argument('--output', default='.', # Default to current directory + help='Output directory for test results') + parser.add_argument('--args', nargs=argparse.REMAINDER, + help='Global arguments to pass to all executables') + args = parser.parse_args() + + # Create output directory if it doesn't exist + if args.output: + os.makedirs(args.output, exist_ok=True) + + # Load arguments configuration + args_config = load_args_config(args.config) + + executables = find_executables(args.dir) + if not executables: + print("No executables found!") + return 1 + + print(f"Found {len(executables)} executables") + + failed = [] + for exe in executables: + ret_code = run_test(exe, args.output, args_config, args.args) + if ret_code != 0: + failed.append((exe.name, ret_code)) + + # Print summary + print("\nTest Summary:") + print(f"Ran {len(executables)} tests") + if failed: + print(f"Failed tests ({len(failed)}):") + for name, code in failed: + print(f" {name}: returned {code}") + return failed[0][1] # Return first failure code + else: + print("All tests passed!") + return 0 + +if __name__ == '__main__': + sys.exit(main()) diff --git a/test_args.json b/test_args.json new file mode 100644 index 00000000..04b79549 --- /dev/null +++ b/test_args.json @@ -0,0 +1,329 @@ +{ + "simpleCUDA2GL": { + "skip": true + }, + "simpleTexture3D": { + "args": [ + "--file=ref_texture3D.bin" + ] + }, + "ptxgen": { + "args": [ + "test.ll", + "-arch=compute_75" + ] + }, + "volumeRender": { + "args": [ + "--file=ref_volume.ppm" + ] + }, + "fluidsGL": { + "skip": true + }, + "simpleD3D11Texture": { + "skip": true + }, + "recursiveGaussian": { + "runs": [ + { + "args": [ + "-sigma=10", + "-file=data/ref_10.ppm" + ] + }, + { + "args": [ + "-sigma=14", + "-file=data/ref_14.ppm" + ] + }, + { + "args": [ + "-sigma=18", + "-file=data/ref_18.ppm" + ] + }, + { + "args": [ + "-sigma=22", + "-file=data/ref_22.ppm" + ] + } + ] + }, + "simpleGL": { + "args": [ + "-file=data/ref_simpleGL.bin" + ] + }, + "bicubicTexture": { + "runs": [ + { + "args": [ + "-mode=0", + "-file=data/0_nearest.ppm" + ] + }, + { + "args": [ + "-mode=1", + "-file=data/1_bilinear.ppm" + ] + }, + { + "args": [ + "-mode=2", + "-file=data/2_bicubic.ppm" + ] + }, + { + "args": [ + "-mode=3", + "-file=data/3_fastcubic.ppm" + ] + }, + { + "args": [ + "-mode=4", + "-file=data/4_catmull-rom.ppm" + ] + } + ] + }, + "simpleVulkan": { + "skip": true + }, + "smokeParticles": { + "args": [ + "-qatest" + ] + }, + "Mandelbrot": { + "runs": [ + { + "args": [ + "-mode=0", + "-file=data/Mandelbrot_fp32.ppm" + ] + }, + { + "args": [ + "-mode=1", + "-file=data/referenceJulia_fp32.ppm" + ] + } + ] + }, + "vulkanImageCUDA": { + "skip": true + }, + "SobelFilter": { + "runs": [ + { + "args": [ + "-mode=0", + "-file=data/ref_orig.pgm" + ] + }, + { + "args": [ + "-mode=1", + "-file=data/ref_tex.pgm" + ] + }, + { + "args": [ + "-mode=2", + "-file=data/ref_shared.pgm" + ] + } + ] + }, + "bilateralFilter": { + "runs": [ + { + "args": [ + "-radius=5", + "-file=data/ref_05.ppm" + ] + }, + { + "args": [ + "-radius=6", + "-file=data/ref_06.ppm" + ] + }, + { + "args": [ + "-radius=7", + "-file=data/ref_07.ppm" + ] + }, + { + "args": [ + "-radius=8", + "-file=data/ref_08.ppm" + ] + } + ] + }, + "nbody": { + "args": [ + "-benchmark", + "-compare", + "-cpu" + ] + }, + "volumeFiltering": { + "args": [ + "-file=data/ref_volumefilter.ppm" + ] + }, + "simpleVulkanMMAP": { + "skip": true + }, + "postProcessGL": { + "skip": true + }, + "marchingCubes": { + "runs": [ + { + "args": [ + "-dump=0", + "-file=data/posArray.bin" + ] + }, + { + "args": [ + "-dump=1", + "-file=data/normalArray.bin" + ] + }, + { + "args": [ + "-dump=2", + "-file=data/compVoxelArray.bin" + ] + } + ] + }, + "bindlessTexture": { + "args": [ + "-file=data/ref_bindlessTexture.bin" + ] + }, + "cuSolverSp_LinearSolver": { + "runs": [ + { + "args": [ + "-R=qr" + ] + }, + { + "args": [ + "-R=chol" + ] + }, + { + "args": [ + "-R=qr", + "-P=symamd" + ] + }, + { + "args": [ + "-R=chol", + "-P=symamd" + ] + }, + { + "args": [ + "-R=lu", + "-P=symamd" + ] + } + ] + }, + "randomFog": { + "args": [ + "-qatest" + ] + }, + "oceanFFT": { + "args": [ + "-qatest" + ] + }, + "FunctionPointers": { + "runs": [ + { + "args": [ + "-mode=0", + "-file=data/ref_orig.pgm" + ] + }, + { + "args": [ + "-mode=1", + "-file=data/ref_tex.pgm" + ] + }, + { + "args": [ + "-mode=2", + "-file=data/ref_shared.pgm" + ] + } + ] + }, + "particles": { + "args": [ + "-file=data/ref_particles.bin" + ] + }, + "imageDenoising": { + "runs": [ + { + "args": [ + "-kernel=0", + "-file=data/ref_passthru.ppm" + ] + }, + { + "args": [ + "-kernel=1", + "-file=data/ref_knn.ppm" + ] + }, + { + "args": [ + "-kernel=2", + "-file=data/ref_nlm.ppm" + ] + }, + { + "args": [ + "-kernel=3", + "-file=data/ref_nlm2.ppm" + ] + } + ] + }, + "boxFilter": { + "runs": [ + { + "args": [ + "-radius=14", + "-file=data/ref_14.ppm" + ] + }, + { + "args": [ + "-radius=22", + "-file=data/ref_22.ppm" + ] + } + ] + } +}