File: //bin/nvidia-bug-report.sh
#!/bin/sh
#
# SPDX-FileCopyrightText: Copyright (c) 2004-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: MIT
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the "Software"),
# to deal in the Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.
#
PATH="/sbin:/usr/sbin:$PATH"
BASE_LOG_FILENAME="nvidia-bug-report.log"
# check if gzip is present
GZIP_CMD=`which gzip 2> /dev/null | head -n 1`
if [ $? -eq 0 -a "$GZIP_CMD" ]; then
GZIP_CMD="gzip -c"
else
GZIP_CMD="cat"
fi
DPY="$DISPLAY"
[ "$DPY" ] || DPY=":0"
set_filename() {
if [ "$GZIP_CMD" = "gzip -c" ]; then
LOG_FILENAME="$BASE_LOG_FILENAME.gz"
OLD_LOG_FILENAME="$BASE_LOG_FILENAME.old.gz"
else
LOG_FILENAME=$BASE_LOG_FILENAME
OLD_LOG_FILENAME="$BASE_LOG_FILENAME.old"
fi
}
query_order=""
separator_single="--------------------------------------------------------------------------------"
separator_double="================================================================================"
# Create temporary files to store skip and error summary entries.
SKIP_TMP=$(mktemp /tmp/nvidia_bug_report_skip.XXXXXX)
ERROR_TMP=$(mktemp /tmp/nvidia_bug_report_error.XXXXXX)
store_result() {
description="$1" # e.g., "NVIDIA GPU Details"
details="$2" # Details to display
# Create a sanitized version of the description for variable names
desc_sanitized=`echo "$description" | tr -c '[:alnum:]_' '_'`
# We'll store "SANITIZED|ORIGINAL" on one line (single-char delimiter).
line_item="$desc_sanitized|$description"
query_order="$query_order
$line_item"
eval "results_details_${desc_sanitized}=\"\$details\""
}
store_skip() {
description="$1"
details="$2"
description=$(echo "$description" | tr '\n' ' ')
details=$(echo "$details" | tr '\n' ' ')
echo "$description|$details" >> "$SKIP_TMP"
}
store_error() {
description="$1"
details="$2"
# Trim any leading/trailing whitespace from resolution.
resolution=$(echo "$3" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')
description=$(echo "$description" | tr '\n' ' ')
details=$(echo "$details" | tr '\n' ' ')
resolution=$(echo "$resolution" | tr '\n' ' ')
printf "%s|%s|%s\n" "$description" "$details" "$resolution" >> "$ERROR_TMP"
}
report_skip() {
description="$1"
details="$2"
{
echo "____________________________________________"
echo ""
echo "Skipping: $description"
echo "$details"
echo ""
} | $GZIP_CMD >> "$LOG_FILENAME"
store_skip "$description" "$details"
}
report_and_store_error() {
description="$1"
details="$2"
exit_block="$3" # Optional parameter to indicate whether to exit the block
resolution="$4"
if [ -z "$resolution" ]; then
resolution="Contact NVIDIA for support"
fi
store_error "$description" "$details" "$resolution"
{
echo "____________________________________________"
echo ""
echo "Error: $description"
echo "$details"
echo "Resolution: $resolution"
echo ""
} | $GZIP_CMD >> "$LOG_FILENAME"
if [ "$exit_block" = "1" ]; then
echo "Exiting current block due to critical error: $description" | $GZIP_CMD >> "$LOG_FILENAME"
exit 1 # Exit the subshell to stop the current block
fi
}
print_skips_table() {
col1_header="Skipped Component"
col1_width=35
printf "%-${col1_width}s | Details\n" "$col1_header"
echo "$separator_double"
DETAIL_WIDTH=60
while IFS='|' read -r description details; do
[ -z "$description" ] && continue
wrapped=$(printf '%s\n' "$details" | fold -s -w $DETAIL_WIDTH)
first_line=$(printf '%s\n' "$wrapped" | sed -n '1p')
printf "%-${col1_width}s | %s\n" "$description" "$first_line"
rest_lines=$(printf '%s\n' "$wrapped" | sed '1d')
if [ -n "$rest_lines" ]; then
echo "$rest_lines" | while IFS= read -r detail_line; do
printf "%-${col1_width}s | %s\n" "" "$detail_line"
done
fi
echo "$separator_single"
done < "$SKIP_TMP"
}
print_errors_table() {
col1_width=35
col2_width=60
col3_width=20
header=$(printf "%-${col1_width}s | %-${col2_width}s | %-${col3_width}s" "Error Component" "Details" "Resolution")
sep_line=$(printf '%*s' $(expr $col1_width + $col2_width + $col3_width + 6) '' | tr ' ' '=')
echo "$header"
echo "$sep_line"
while IFS='|' read -r comp details resolution; do
[ -z "$comp" ] && continue
comp_wrapped=$(printf '%s\n' "$comp" | fold -s -w $col1_width)
details_wrapped=$(printf '%s\n' "$details" | fold -s -w $col2_width)
resolution_wrapped=$(printf '%s\n' "$resolution" | fold -s -w $col3_width)
comp_lines=$(printf '%s\n' "$comp_wrapped")
details_lines=$(printf '%s\n' "$details_wrapped")
resolution_lines=$(printf '%s\n' "$resolution_wrapped")
comp_count=$(printf '%s\n' "$comp_lines" | wc -l | sed 's/ //g')
details_count=$(printf '%s\n' "$details_lines" | wc -l | sed 's/ //g')
resolution_count=$(printf '%s\n' "$resolution_lines" | wc -l | sed 's/ //g')
max_lines=$(printf "%s\n" "$comp_count" "$details_count" "$resolution_count" | sort -nr | head -n 1)
i=1
while [ $i -le $max_lines ]; do
comp_line=$(printf '%s\n' "$comp_lines" | sed -n "${i}p")
details_line=$(printf '%s\n' "$details_lines" | sed -n "${i}p")
resolution_line=$(printf '%s\n' "$resolution_lines" | sed -n "${i}p")
printf "%-${col1_width}s | %-${col2_width}s | %-${col3_width}s\n" "$comp_line" "$details_line" "$resolution_line"
i=$(expr $i + 1)
done
sep_line_single=$(printf '%*s' $(expr $col1_width + $col2_width + $col3_width + 6) '' | tr ' ' '-')
echo "$sep_line_single"
done < "$ERROR_TMP"
}
print_versions_table() {
order_list="$query_order"
col1_header="Component"
col1_width=35
# Print header
printf "%-${col1_width}s | Details\n" "$col1_header"
echo "$separator_double"
echo "$order_list" | while IFS= read -r line
do
# Skip blank lines
[ -z "$line" ] && continue
# Extract sanitized + original description
sanitized="$(printf '%s' "$line" | cut -f1 -d'|')"
original="$(printf '%s' "$line" | cut -f2- -d'|')"
# Retrieve details from dynamic variables
eval details="\$results_details_${sanitized}"
# If details is empty, print "None" in the details column
if [ -z "$details" ]; then
details="None"
fi
# Split details into first line + rest
first_line="$(printf '%s\n' "$details" | sed -n '1p')"
printf "%-${col1_width}s | %s\n" "$original" "$first_line"
rest_lines="$(printf '%s\n' "$details" | sed '1d')"
if [ -n "$rest_lines" ]; then
echo "$rest_lines" | while IFS= read -r detail_line
do
printf "%-${col1_width}s | %s\n" "" "$detail_line"
done
fi
echo "$separator_single"
done
}
parse_ibv_devinfo() {
dev="$1"
shift
requested_fields="$*"
devinfo="$(ibv_devinfo -d "$dev" 2>/dev/null)"
hca_id="$(echo "$devinfo" | grep -m1 '^hca_id:' | awk '{print $2}')"
fw_ver="$(echo "$devinfo" | grep -m1 'fw_ver:' | awk '{print $2}')"
link_layer="$(echo "$devinfo" | grep -m1 'link_layer:' | awk '{print $2}')"
state="$(echo "$devinfo" | grep -m1 'state:' | sed -E 's/.*state:\s+([^ ]+).*/\1/')"
output=""
for field in $requested_fields
do
case "$field" in
device_name)
if [ "$dev" != "$hca_id" ]; then
output="$output""Device Name : $hca_id
"
fi
;;
fw_ver)
output="$output""Firmware Ver: $fw_ver
"
;;
link_layer)
output="$output""Link Layer : $link_layer
"
;;
state)
output="$output""Port State : $state
"
;;
esac
done
printf "%s" "$output"
}
run_query_ibv_devinfo() {
description="$1" # e.g., "InfiniBand Firmware Versions"
# Is ibv_devinfo present?
if ! command -v ibv_devinfo >/dev/null 2>&1; then
store_result "$description" "None"
return
fi
# Gather devices
devices="$(ibv_devinfo -l 2>/dev/null | sed -n 's/^[[:space:]]*\(mlx5_[0-9]*\)$/\1/p')"
# Parse each device
desired_fields="device_name fw_ver link_layer state"
aggregated_output=""
for dev in $devices
do
dev_output="$(parse_ibv_devinfo "$dev" "$desired_fields")"
aggregated_output="$aggregated_output""Device : $dev
$dev_output
"
done
# Store into our table
store_result "$description" "$aggregated_output"
}
check_command() {
cmd="$1" # Command to check
description="$2" # Description for the table entry
# Check if the command exists
if ! command -v "$cmd" >/dev/null 2>&1; then
# If the command is not found, log it to the table
store_result "$description" "$cmd command not found"
return 1
fi
return 0
}
# Function to detect if MODs driver is loaded vs standard RM driver
detect_driver_type() {
# Use timeout to prevent hangs, and check /proc/modules as fallback
if command -v timeout >/dev/null 2>&1; then
# Use timeout command if available
if timeout 5 lsmod 2>/dev/null | grep -q "^mods "; then
echo "Running MODs Driver on GPUs"
elif timeout 5 lsmod 2>/dev/null | grep -q "^nvidia "; then
echo "Running RM Driver on GPUs"
else
echo "GPU Driver not installed?"
fi
else
# Fallback: check /proc/modules directly (safer than lsmod)
if [ -r /proc/modules ]; then
if grep -q "^mods " /proc/modules 2>/dev/null; then
echo "Running MODs Driver on GPUs"
elif grep -q "^nvidia " /proc/modules 2>/dev/null; then
echo "Running RM Driver on GPUs"
else
echo "GPU Driver not installed?"
fi
else
echo "GPU Driver not installed?"
fi
fi
}
# Function to discover available IB devices
discover_ib_devices() {
ib_devices=""
fallback_devices=""
# Check if /sys/class/infiniband directory exists
if [ ! -d "/sys/class/infiniband" ]; then
return 1
fi
# Loop through each InfiniBand device directory
for dir in /sys/class/infiniband/*/device; do
# Check if the device directory exists
if [ -d "$dir" ]; then
# Define the path to the VPD file
vpd_file="$dir/vpd"
# Check if the VPD file exists and contains SW_MNG (NVL5+ system)
if [ -f "$vpd_file" ] && grep -q "SW_MNG" "$vpd_file" 2>/dev/null; then
# Extract the InfiniBand device name using parameter expansion
device_name="${dir%/device}" # Removes '/device' from the end of $dir
device_name="${device_name##*/}" # Extracts the part after the last '/'
# Add to fallback list (all SW_MNG devices with non-empty names)
if [ -n "$device_name" ]; then
fallback_devices="$fallback_devices $device_name"
# Check if pkey file exists for this device
pkey_file="/sys/class/infiniband/$device_name/ports/1/pkeys/0"
if [ -f "$pkey_file" ]; then
pkey_value=$(cat "$pkey_file" 2>/dev/null)
# Only include devices that are full members (0xffff) in primary list
if [ "$pkey_value" = "0xffff" ]; then
ib_devices="$ib_devices $device_name"
fi
fi
fi
fi
fi
done
# Remove leading spaces
ib_devices=$(echo "$ib_devices" | sed 's/^[[:space:]]*//')
fallback_devices=$(echo "$fallback_devices" | sed 's/^[[:space:]]*//')
# If no full member devices found, fall back to all SW_MNG devices
if [ -z "$ib_devices" ] && [ -n "$fallback_devices" ]; then
ib_devices="$fallback_devices"
fi
# If still no devices, fall back to default mlx5_0 if it exists
if [ -z "$ib_devices" ] && [ -d "/sys/class/infiniband/mlx5_0" ]; then
ib_devices="mlx5_0"
fi
echo "$ib_devices"
}
run_query_mlxlink() {
description="$1" # e.g., "Mellanox Link Firmware Versions"
query="$2" # Query string to match in 'mst status'
# Check if required commands exist
if ! check_command "mst" "$description"; then return; fi
if ! check_command "mlxlink" "$description"; then return; fi
# Discover available IB devices
ib_devices=$(discover_ib_devices)
if [ -z "$ib_devices" ]; then
store_result "$description" "No IB devices found"
return
fi
# Run mst status and capture output
mst_output=$(sudo mst status -v 2>&1)
if [ $? -ne 0 ]; then
store_result "$description" "Error running 'mst status': $mst_output"
return
fi
# Initialize result aggregation
aggregated_output=""
# Process each discovered IB device
for ib_device in $ib_devices; do
while IFS= read -r tmp_line; do
echo "$tmp_line" | grep "$query" | grep "$ib_device" >/dev/null 2>&1
if [ $? -eq 0 ]; then
# Extract device name
devName=$(echo "$tmp_line" | awk -F" " '{print $1}')
mlxlink_output=$(sudo mlxlink -d "$devName" 2>&1)
if [ $? -ne 0 ]; then
aggregated_output="$aggregated_output""Device: $devName
Error: $(echo "$mlxlink_output" | head -n 1)
"
continue
fi
aggregated_output="$aggregated_output""Device: $devName
$mlxlink_output
"
fi
done <<EOF
$(echo "$mst_output" | grep "$query" | grep "$ib_device")
EOF
done
# Store the result
if [ -z "$aggregated_output" ]; then
store_result "$description" "No devices matching query '$query'"
else
store_result "$description" "$aggregated_output"
fi
}
run_query() {
cmd="$1"
args="$2"
description="$3"
# If command is not found
if ! command -v "$cmd" >/dev/null 2>&1; then
store_result "$description" "None"
return
fi
# Capture all stdout+stderr
output=`$cmd $args 2>&1`
ret_code=$?
if [ "$description" = "OS Details" ] && [ "$ret_code" -eq 0 ]; then
os_details=""
if command -v lsb_release >/dev/null 2>&1; then
os_details="$os_details""Distribution : $(lsb_release -d -s)
"
elif [ -f /etc/os-release ]; then
distline="$(grep '^PRETTY_NAME=' /etc/os-release | cut -d= -f2- | tr -d '"')"
if [ -n "$distline" ]; then
os_details="$os_details""Distribution : $distline
"
else
os_details="$os_details""Distribution : Unknown
"
fi
else
os_details="$os_details""Distribution : Unknown
"
fi
os_details="$os_details""Kernel : $(uname -r)
"
os_details="$os_details""Hostname : $(hostname)
"
os_details="$os_details""Architecture : $(uname -m)
"
os_details="$os_details""Uptime : $(uptime -p 2>/dev/null || true)
"
output="$os_details"
fi
# Mark success or failure
if [ "$ret_code" -eq 0 ]; then
store_result "$description" "$output"
else
first_line="$(echo "$output" | head -n 1)"
store_result "$description" "Failed: $first_line"
fi
}
run_query_file() {
file_path="$1"
pattern="$2"
description="$3"
# Check if file is readable
if [ ! -r "$file_path" ]; then
store_result "$description" "None"
return
fi
# Grab matching lines
filtered_output=`grep "$pattern" "$file_path" 2>/dev/null`
ret_code=$?
if [ $ret_code -eq 0 ]; then
if [ -z "$filtered_output" ]; then
store_result "$description" "None"
else
store_result "$description" "$filtered_output"
fi
else
store_result "$description" "None"
fi
}
fetch_software_firmware_versions() {
run_query "vulkaninfo" "--version" "Vulkan Info"
# Detect driver type (MODs vs RM)
driver_type=$(detect_driver_type)
# Skip nvidia-smi calls in safe mode to avoid hangs when driver is not responding
if [ $BUG_REPORT_SAFE_MODE -eq 0 ]; then
run_query "nvidia-smi" "--version" "NVIDIA SMI"
run_query "nvidia-smi" "--query-gpu=name,driver_version,memory.total,vbios_version,pci.bus_id,serial --format=csv,noheader" \
"NVIDIA GPU Details"
else
# In safe mode, try to get driver version from modinfo as fallback
if command -v modinfo >/dev/null 2>&1; then
driver_version=$(modinfo nvidia 2>/dev/null | grep "^version:" | awk '{print $2}')
if [ -n "$driver_version" ]; then
store_result "NVIDIA SMI" "Driver version (from modinfo): $driver_version"
else
store_result "NVIDIA SMI" "Unable to fetch (safe mode - driver may be hung)"
fi
else
store_result "NVIDIA SMI" "Unable to fetch (safe mode - driver may be hung)"
fi
# Provide specific messaging for MODs driver
if echo "$driver_type" | grep -q "MODs Driver"; then
store_result "NVIDIA GPU Details" "Skipped in safe mode (MODs driver detected - nvidia-smi may hang)"
else
store_result "NVIDIA GPU Details" "Skipped in safe mode (driver may be hung)"
fi
fi
run_query "nvidia-settings" "--version" "NVIDIA Settings"
run_query "nv-fabricmanager" "--version" "NVIDIA Fabric Manager"
run_query "opensm" "--version" "NVIDIA Subnet Manager"
run_query "mlxlink" "--version" "Mellanox Link"
run_query "ibstat" "--version" "InfiniBand Status"
run_query "ibnetdiscover" "--version" "InfiniBand Network Discovery"
run_query_file "/var/log/dmesg" "MSE\|uCode\|NETIR" "NVIDIA MSE/NETIR Versions"
run_query_mlxlink "NVIDIA Switch Details" "Quantum"
run_query_ibv_devinfo "NVIDIA NIC Details"
run_query "lsb_release" "-d" "OS Details"
print_versions_table
}
module_names="nvidia nvidia_drm nvidia_modeset nvidia_uvm nvidia_peermem"
usage_bug_report_message() {
echo "Please include the '$LOG_FILENAME' log file when reporting"
echo "your bug via the NVIDIA Linux forum (see forums.developer.nvidia.com)"
echo "or by sending email to 'linux-bugs@nvidia.com'."
echo ""
echo "By delivering '$LOG_FILENAME' to NVIDIA, you acknowledge"
echo "and agree that personal information may inadvertently be included in"
echo "the output. Notwithstanding the foregoing, NVIDIA will use the"
echo "output only for the purpose of investigating your reported issue."
}
usage() {
echo ""
echo "$(basename $0): NVIDIA Linux Graphics Driver bug reporting shell script."
echo ""
usage_bug_report_message
echo "Usage:"
echo " $(basename $0) [OPTIONS]"
echo ""
echo "Options:"
echo " -h, --help"
echo " Print this help message and exit."
echo ""
echo " --output-file <file>"
echo " Write output to <file>. If 'gzip' is available, the file will"
echo " automatically be compressed to <file>.gz."
echo " Default: nvidia-bug-report.log(.gz)."
echo ""
echo " --safe-mode"
echo " Disable certain queries that might hang the system. Useful if you"
echo " experience freezes, high CPU usage, or suspect problematic kernel modules."
echo ""
echo " --extra-system-data"
echo " Gather additional system information and CPU backtraces for deeper"
echo " debugging. Consider using this if the script hangs without --safe-mode"
echo " or you need more extensive kernel data."
echo ""
#VGX_SPECIFIC_CODE_START
if [ -n "$IS_XEN_HYPERVISOR" ]; then
echo " --domain-name \"<domain_name>\""
echo " Generate a bug report for the specified Xen virtual machine domain."
echo " Only valid if running within Xen dom0."
echo ""
fi
#VGX_SPECIFIC_CODE_END
echo " -v, --version"
echo " Show the current version of nvidia-bug-report.sh and exit."
echo " -V, --versions"
echo " Show the current versions of NVIDIA software and firmware on the node."
echo ""
echo "Description:"
echo " This script collects logs and configuration details—covering kernel"
echo " messages, GPU driver info, Xorg logs, hypervisor data (if applicable),"
echo " and more—to help diagnose NVIDIA driver or GPU-related issues. Run it"
echo " as root or with sudo for the most complete data."
echo ""
echo " If certain tools (e.g., 'nvidia-smi', 'lspci', 'mstflint') are missing,"
echo " the script will note \"Skipping...\" in the final report but continue"
echo " collecting other data."
echo ""
echo "Troubleshooting Tips:"
echo " - If the script hangs, try using '--safe-mode'."
echo " - If you need deeper system info (e.g., CPU backtraces), use '--extra-system-data'."
echo " - Check final '.log' or '.log.gz' for 'Skipping...' lines to see if any"
echo " dependencies were missing or commands failed."
echo " - Ensure you have installed recommended dependencies (see the full documentation"
echo " or 'system_requirements.rst'). Official releases of the NVIDIA driver/firmware"
echo " reduce hangs and incomplete captures."
echo ""
echo "Examples:"
echo " 1) sudo ./nvidia-bug-report.sh"
echo " Creates 'nvidia-bug-report.log.gz' if 'gzip' is installed."
echo ""
echo " 2) sudo ./nvidia-bug-report.sh --safe-mode"
echo " Skips queries known to cause hangs on certain systems."
echo ""
echo " 3) sudo ./nvidia-bug-report.sh --extra-system-data --output-file custom_report"
echo " Gathers extended kernel info into 'custom_report.log.gz' (compressed if possible)."
echo ""
}
NVIDIA_BUG_REPORT_CHANGE='$Change: 38278083 $'
NVIDIA_BUG_REPORT_VERSION=`echo "$NVIDIA_BUG_REPORT_CHANGE" | tr -c -d "[:digit:]"`
# Set the default filename so that it won't be empty in the usage message
set_filename
# Parse arguments: Optionally set output file, run in safe mode, include extra
# system data, or print help
BUG_REPORT_SAFE_MODE=0
BUG_REPORT_EXTRA_SYSTEM_DATA=0
SAVED_FLAGS=$@
while [ "$1" != "" ]; do
case "$1" in
-o | --output-file )
if [ -z "$2" ]; then
echo "Error: --output-file requires a filename argument."
usage
exit 1
elif [ "$(echo "$2" | cut -c 1)" = "-" ]; then
echo "Warning: Questionable filename \"$2\": possible missing argument?"
fi
BASE_LOG_FILENAME="$2"
# Override the default filename
set_filename
shift
;;
--safe-mode )
BUG_REPORT_SAFE_MODE=1
;;
--extra-system-data )
BUG_REPORT_EXTRA_SYSTEM_DATA=1
;;
-h | --help )
usage
exit 0
;;
-v | --version )
echo "${NVIDIA_BUG_REPORT_VERSION}"
exit 0
;;
-V | --versions )
fetch_software_firmware_versions
exit 0
;;
* )
echo "Error: Unknown option \"$1\"."
usage
exit 1
;;
esac
shift
done
#
# echo_metadata() - echo metadata of specified file
#
echo_metadata() {
printf "*** ls: "
/bin/ls -l --full-time "$1" 2> /dev/null
if [ $? -ne 0 ]; then
# Run dumb ls -l. We might not get one-second mtime granularity, but
# that is probably okay.
ls -l "$1" 2>&1
fi
}
#
# report_file() - print a report for the specified file to stdout: if it
# exists, dumps metadata and contents.
#
report_file() {
echo "____________________________________________"
echo ""
if [ ! -f "$1" ]; then
echo "*** $1 does not exist"
elif [ ! -r "$1" ]; then
echo "*** $1 is not readable"
else
echo "*** $1"
echo_metadata "$1"
cat "$1"
fi
echo ""
}
#
# append() - append the contents of the specified file to the log
#
append() {
report_file "$1" | $GZIP_CMD >> $LOG_FILENAME
}
#
# append_silent() - same as append(), but don't print anything
# if the file does not exist
#
append_silent() {
(
if [ -f "$1" -a -r "$1" ]; then
echo "____________________________________________"
echo ""
echo "*** $1"
echo_metadata "$1"
cat "$1"
echo ""
fi
) | $GZIP_CMD >> $LOG_FILENAME
}
#
# append_glob() - use the shell to expand a list of files, and invoke
# report_file for each of them; append the result to the log
#
append_glob() {
for append_glob_iterator in `ls $1 2> /dev/null;`; do
report_file "$append_glob_iterator"
done | $GZIP_CMD >> $LOG_FILENAME
}
#
# append_file_or_dir_silent() - if $1 is a regular file, append it; otherwise,
# if $1 is a directory, append all files under it. Don't print anything if the
# file does not exist.
#
append_file_or_dir_silent() {
if [ -f "$1" ]; then
append "$1"
elif [ -d "$1" ]; then
append_glob "$1/*"
fi
}
#
# append_binary_file() - Encode a binary file into a ascii string format
# using 'base64' and append the contents output to the log file
#
append_binary_file() {
(
base64=`which base64 2> /dev/null | head -n 1`
if [ $? -eq 0 -a -x "$base64" ]; then
if [ -f "$1" -a -r "$1" ]; then
echo "____________________________________________"
echo ""
echo "base64 \"$1\""
echo ""
base64 "$1" 2> /dev/null
echo ""
fi
else
report_skip "$1 output" "base64 not found"
echo ""
fi
) | $GZIP_CMD >> $LOG_FILENAME
}
#
# report_command() - print the output of the specified command to stdout
#
report_command() {
if [ -n "$1" ]; then
echo "$1"
echo ""
$1 2>&1
echo ""
fi
}
#
# search_string_in_logs() - search for string $2 in log file $1
#
search_string_in_logs() {
if [ -f "$1" ]; then
echo ""
if [ -r "$1" ]; then
echo " $1:"
grep $2 "$1" 2> /dev/null
return 0
else
echo "$1 is not readable"
fi
fi
return 1
}
#
# print_package_for_file() - Print the package that owns the file $1
#
print_package_for_file()
{
# Try to figure out which package manager we should use, and print which
# package owns a file.
pkgcmd=`which dpkg-query 2> /dev/null | head -n 1`
if [ $? -eq 0 -a -n "$pkgcmd" ]; then
pkgoutput=`"$pkgcmd" --search "$1" 2> /dev/null`
if [ $? -ne 0 -o "x$pkgoutput" = "x" ] ; then
echo No package found for $1
return
fi
pkgname=$(echo "$pkgoutput" | sed -e 's/:[[:space:]].*//')
if [ "x$pkgname" = "x" ] ; then
echo Can\'t parse package result: $pkgoutput
return
fi
"$pkgcmd" --show --showformat=' Package: ${Package}:${Architecture} ${Version}\n' $pkgname
return
fi
pkgcmd=`which pacman 2> /dev/null | head -n 1`
if [ $? -eq 0 -a -n "$pkgcmd" ]; then
pkgoutput=`"$pkgcmd" --query --owns "$1" 2> /dev/null`
if [ $? -ne 0 -o "x$pkgoutput" = "x" ] ; then
echo No package found for $1
return
fi
echo "$pkgoutput"
return
fi
pkgcmd=`which rpm 2> /dev/null | head -n 1`
if [ $? -eq 0 -a -n "$pkgcmd" ]; then
"$pkgcmd" -q -f "$1" 2> /dev/null
return
fi
}
GLVND_HELPER_BASE_PATH=/usr/lib/nvidia
#
# print_libglvnd_library() - Print information about a libglvnd library.
# $1 is the path to where the helper program and libraries are.
# $2 is the API to check
# $3 is the name of the library to look for
#
print_libglvnd_library()
{
echo Checking library: $3
__GLX_VENDOR_LIBRARY_NAME=installcheck
__EGL_VENDOR_LIBRARY_FILENAMES=$GLVND_HELPER_BASE_PATH/egl_dummy_vendor.json
export __GLX_VENDOR_LIBRARY_NAME
export __EGL_VENDOR_LIBRARY_FILENAMES
result=`LD_LIBRARY_PATH="$1${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" $1/glvnd_check $2 $3`
code=$?
unset __GLX_VENDOR_LIBRARY_NAME
unset __EGL_VENDOR_LIBRARY_FILENAMES
case $code in
0)
echo Found compatible libglvnd library $3
;;
1)
echo Found non-libglvnd library $3
;;
2)
echo Found incompatible libglvnd library $3
;;
3)
echo Library $3 does not exist
return $code
;;
*)
echo Internal error when checking $3
echo "$result"
return 4
;;
esac
libpath=`echo "$result" | grep "^PATH " | cut -s "-d " -f2-`
echo Found library at: "$libpath"
info=`echo "$result" | grep "^LIBGLVND_ABI " | cut -s "-d " -f2,3`
if [ -n "$info" ] ; then
echo Libglvnd ABI version: $info
fi
info=`echo "$result" | grep "^CLIENT_STRING " | cut -s "-d " -f2-`
if [ -n "$info" ] ; then
echo Client version/vendor string: "$info"
fi
print_package_for_file $libpath
echo -----
return $code
}
#
# Start of script
#
# check that we are root (needed for `lspci -vxxxx` and potentially for
# accessing kernel log files)
if [ `id -u` -ne 0 ]; then
echo "ERROR: Please run $(basename $0) as root."
exit 1
fi
# move any old log file (zipped) out of the way
if [ -f $LOG_FILENAME ]; then
mv $LOG_FILENAME $OLD_LOG_FILENAME
fi
# make sure what we can write to the log file
touch $LOG_FILENAME 2> /dev/null
if [ $? -ne 0 ]; then
echo
echo "ERROR: Working directory is not writable; please cd to a directory"
echo " where you have write permission so that the $LOG_FILENAME"
echo " file can be written."
echo
exit 1
fi
# Make sure we can create temp files. Check TMPDIR (POSIX-compliant) if present
[ -z "$TMPDIR" ] && TMPDIR="/tmp"
tmptest="$TMPDIR/nvbrtesttemp.$$"
touch "$tmptest"
if [ $? -ne 0 ]; then
echo "ERROR: The directory $TMPDIR is not writable. Please set TMPDIR"
echo " to a directory with write permission for temporary output files"
exit 1
else
rm -f "$tmptest"
fi
# print a start message to stdout
echo ""
echo "nvidia-bug-report.sh will now collect information about your"
echo "system and create the file '$LOG_FILENAME' in the current"
echo "directory. It may take several seconds to run. In some"
echo "cases, it may hang trying to capture data generated dynamically"
echo "by the Linux kernel and/or the NVIDIA kernel module. While"
echo "the bug report log file will be incomplete if this happens, it"
echo "may still contain enough data to diagnose your problem."
echo ""
if [ $BUG_REPORT_SAFE_MODE -eq 0 ]; then
echo "If nvidia-bug-report.sh hangs, consider running with the --safe-mode"
echo "and --extra-system-data command line arguments."
echo ""
fi
usage_bug_report_message
echo ""
echo -n "Running $(basename $0)...";
# print prologue to the log file
(
echo "____________________________________________"
echo ""
echo "Start of NVIDIA bug report log file. Please include this file, along"
echo "with a detailed description of your problem, when reporting a graphics"
echo "driver bug via the NVIDIA Linux forum (see forums.developer.nvidia.com)"
echo "or by sending email to 'linux-bugs@nvidia.com'."
echo ""
echo "nvidia-bug-report.sh Version: $NVIDIA_BUG_REPORT_VERSION"
echo ""
echo "Date: `date`"
echo "uname: `uname -a`"
echo "command line flags: $SAVED_FLAGS"
echo ""
) | $GZIP_CMD >> $LOG_FILENAME
# print software firmware versions
(
fetch_software_firmware_versions
) | $GZIP_CMD >> $LOG_FILENAME
# Add driver type detection information (only if not in safe mode to avoid hangs)
if [ $BUG_REPORT_SAFE_MODE -eq 0 ]; then
# Show driver type on console
driver_type=$(detect_driver_type)
echo ""
echo "Detected driver type: $driver_type"
if echo "$driver_type" | grep -q "MODs Driver"; then
echo ""
echo "WARNING: MODs driver detected. nvidia-smi may hang with MODs driver."
echo "If experiencing hangs, try:"
echo " sudo rmmod mods"
echo " sudo modprobe nvidia"
echo " sudo nvidia-smi # confirm RM driver is working"
echo " then run nvidia-bug-report.sh"
fi
# Also log detailed info to file
(
echo "____________________________________________"
echo ""
echo "NVIDIA Driver Type Detection:"
echo ""
echo "Detected driver type: $driver_type"
if echo "$driver_type" | grep -q "MODs Driver"; then
echo ""
echo "nvidia-smi does not work with MODs driver."
echo "Install RM Driver using these instructions:"
echo " sudo rmmod mods"
echo " sudo modprobe nvidia"
echo " sudo nvidia-smi # confirm RM driver is working"
echo " then run nvidia-bug-report.sh"
elif echo "$driver_type" | grep -q "RM Driver"; then
echo "Standard RM driver detected."
else
echo "No NVIDIA driver modules detected."
fi
echo ""
) | $GZIP_CMD >> $LOG_FILENAME
else
# Show safe mode message on console
echo "Skipped driver type detection (safe mode - driver may be unresponsive)"
# Also log to file
(
echo "____________________________________________"
echo ""
echo "NVIDIA Driver Type Detection:"
echo ""
echo "Skipped in safe mode"
echo ""
) | $GZIP_CMD >> $LOG_FILENAME
fi
if [ $BUG_REPORT_SAFE_MODE -eq 0 ]; then
# List the sysfs entries for all NVIDIA device functions
# This info is useful to debug dynamic power management issues
#
# NOTE: We need to query this before other things in this script,
# because other operations may alter the power management
# state of the GPU(s).
for subdir in `ls /sys/bus/pci/devices/ 2> /dev/null`; do
vendor_id=`cat /sys/bus/pci/devices/$subdir/vendor 2> /dev/null`
if [ "$vendor_id" = "0x10de" ]; then
append "/sys/bus/pci/devices/$subdir/power/control"
append "/sys/bus/pci/devices/$subdir/power/runtime_status"
append "/sys/bus/pci/devices/$subdir/power/runtime_usage"
# Get the parent port's power resources to determine
# if parent port can control the power. We check the parent port
# only for the first subfunction (VGA/3D).
pci_class=`cat /sys/bus/pci/devices/$subdir/class 2> /dev/null`
if [ "$pci_class" = "0x030000" ] || [ "$pci_class" = "0x030200" ]; then
path_to_device=`readlink -f /sys/bus/pci/devices/$subdir`
pr_d3hot="$path_to_device/../firmware_node/power_resources_D3hot"
(
echo "____________________________________________"
echo ""
echo "power_resources_d3hot directory of the parent PCI/e port"
echo ""
echo_metadata $pr_d3hot
) | $GZIP_CMD >> $LOG_FILENAME
fi
fi
done
for GPU in `ls /proc/driver/nvidia/gpus/ 2> /dev/null`; do
append "/proc/driver/nvidia/gpus/$GPU/power"
done
fi
# append OPAL (IBM POWER system firmware) messages
append_silent "/sys/firmware/opal/msglog"
# append useful files
append "/etc/issue"
append_silent "/etc/redhat-release"
append_silent "/etc/redhat_version"
append_silent "/etc/fedora-release"
append_silent "/etc/slackware-release"
append_silent "/etc/slackware-version"
append_silent "/etc/debian_release"
append_silent "/etc/debian_version"
append_silent "/etc/mandrake-release"
append_silent "/etc/yellowdog-release"
append_silent "/etc/sun-release"
append_silent "/etc/release"
append_silent "/etc/gentoo-release"
append_silent "/etc/lsb-release"
append_silent "/etc/os-release"
append_silent "/etc/fastos-release"
append_silent "/etc/fastos-ota"
append_silent "/etc/dgx-release"
append "/var/log/nvidia-installer.log"
append_silent "/var/log/nvidia-uninstall.log"
# find and append all make.log files in /var/lib/dkms for module nvidia
if [ -d "/var/lib/dkms/nvidia" ]; then
for log in `find "/var/lib/dkms/nvidia" -name "make.log"`; do
append $log
done
fi
# check the status of the nvidia-suspend, nvidia-hibernate, nvidia-resume
# and nvidia-powerd systemd services
systemctl=`which systemctl 2> /dev/null | head -n 1`
if [ $? -eq 0 -a -x "$systemctl" ]; then
cmd="$systemctl status nvidia-suspend.service nvidia-hibernate.service nvidia-resume.service nvidia-powerd.service nvidia-persistenced.service"
(
echo "____________________________________________"
echo ""
echo "$cmd"
$cmd
echo ""
) 2> /dev/null | $GZIP_CMD >> $LOG_FILENAME
fi
# use systemd's journalctl to capture X logs where applicable
journalctl=`which journalctl 2> /dev/null | head -n 1`
if [ $? -eq 0 -a -x "$journalctl" ]; then
for match in _COMM=Xorg \
_COMM=Xorg.bin \
_COMM=X \
_COMM=gdm-x-session \
"SYSLOG_IDENTIFIER=systemd-coredump -g nvidia"; do
for boot in -0 -1 -2; do
if journalctl -b $boot -n 1 $match >/dev/null 2>&1; then
(
echo "____________________________________________"
echo ""
echo "journalctl -b $boot $match"
echo ""
journalctl -b $boot $match
echo ""
) 2> /dev/null | $GZIP_CMD >> $LOG_FILENAME
fi
done
done
fi
# use systemd's coredumpctl to capture X coredumps where applicable
coredumpctl=`which coredumpctl 2> /dev/null | head -n 1`
if [ $? -eq 0 -a -x "$coredumpctl" ]; then
cmd="$coredumpctl info COREDUMP_COMM=Xorg COREDUMP_COMM=Xorg.bin COREDUMP_COMM=X"
(
echo "____________________________________________"
echo ""
echo "$cmd"
$cmd
echo ""
) 2> /dev/null | $GZIP_CMD >> $LOG_FILENAME
fi
# append the X log; also, extract the config file named in the X log
# and append it; look for X log files with names of the form:
# /var/log/Xorg.{0,1,2,3,4,5,6,7}.{log,log.old}
xconfig_file_list=
svp_config_file_list=
NEW_LINE="
"
for i in 0 1 2 3 4 5 6 7; do
for log_suffix in log log.old; do
log_filename="/var/log/Xorg.${i}.${log_suffix}"
append_silent "${log_filename}"
# look for the X configuration files/directories referenced by this X log
if [ -f ${log_filename} -a -r ${log_filename} ]; then
config_file=`grep "Using config file" ${log_filename} | cut -f 2 -d \"`
config_dir=`grep "Using config directory" ${log_filename} | cut -f 2 -d \"`
sys_config_dir=`grep "Using system config directory" ${log_filename} | cut -f 2 -d \"`
for j in "$config_file" "$config_dir" "$sys_config_dir"; do
if [ "$j" ]; then
# multiple of the logs we find above might reference the
# same X configuration file; keep a list of which X
# configuration files we find, and only append X
# configuration files we have not already appended
echo "${xconfig_file_list}" | grep ":${j}:" > /dev/null
if [ "$?" != "0" ]; then
xconfig_file_list="${xconfig_file_list}:${j}:"
if [ -d "$j" ]; then
append_glob "$j/*.conf"
else
append "$j"
fi
fi
fi
done
# append NVIDIA 3D Vision Pro configuration settings
svp_conf_files=`grep "Option \"3DVisionProConfigFile\"" ${log_filename} | cut -f 4 -d \"`
if [ "${svp_conf_files}" ]; then
OLD_IFS="$IFS"
IFS=$NEW_LINE
for svp_file in ${svp_conf_files}; do
IFS="$OLD_IFS"
echo "${svp_config_file_list}" | grep ":${svp_file}:" > /dev/null
if [ "$?" != "0" ]; then
svp_config_file_list="${svp_config_file_list}:${svp_file}:"
append_binary_file "${svp_file}"
fi
IFS=$NEW_LINE
done
IFS="$OLD_IFS"
fi
fi
done
done
# Append any config files found in home directories
cat /etc/passwd \
| cut -d : -f 6 \
| sort | uniq \
| while read DIR; do
append_file_or_dir_silent "$DIR/.nv/nvidia-application-profiles-rc"
append_file_or_dir_silent "$DIR/.nv/nvidia-application-profiles-rc.backup"
append_file_or_dir_silent "$DIR/.nv/nvidia-application-profiles-rc.d"
append_file_or_dir_silent "$DIR/.nv/nvidia-application-profiles-rc.d.backup"
append_silent "$DIR/.nv/nvidia-application-profile-globals-rc"
append_silent "$DIR/.nv/nvidia-application-profile-globals-rc.backup"
append_silent "$DIR/.nvidia-settings-rc"
for f in "$DIR/.config/vulkan/icd.d/*" \
"$DIR/.config/vulkansc/icd.d/*" \
"$DIR/.local/share/vulkan/icd.d/*" \
"$DIR/.local/share/vulkansc/icd.d/*"; do
append_silent $f
done
done
# Capture global app profile configs
append_file_or_dir_silent "/etc/nvidia/nvidia-application-profiles-rc"
append_file_or_dir_silent "/etc/nvidia/nvidia-application-profiles-rc.d"
append_file_or_dir_silent /usr/share/nvidia/nvidia-application-profiles-*-rc
# append ldd info
(
echo "____________________________________________"
echo ""
glxinfo=`which glxinfo 2> /dev/null | head -n 1`
if [ $? -eq 0 -a -x "$glxinfo" ]; then
echo "ldd $glxinfo"
echo ""
ldd $glxinfo 2> /dev/null
echo ""
else
report_skip "ldd output" "glxinfo not found"
echo ""
fi
) | $GZIP_CMD >> $LOG_FILENAME
# append Vulkan ICD info
(
echo "____________________________________________"
echo ""
vkinfo=`ldconfig -N -v -p 2> /dev/null | /bin/grep libvulkan.so.1 | awk 'NF>1{print $NF}'`
if [ $? -eq 0 -a -n "$vkinfo" ]; then
echo "Found Vulkan loader(s):"
readlink -f ${vkinfo} 2> /dev/null
echo ""
# See https://github.com/KhronosGroup/Vulkan-Loader/blob/main/docs/LoaderDriverInterface.md
echo "Listing common ICD paths:"
ls -d /usr/local/etc/vulkan/icd.d/* 2> /dev/null
ls -d /usr/local/share/vulkan/icd.d/* 2> /dev/null
ls -d /etc/vulkan/icd.d/* 2> /dev/null
ls -d /usr/share/vulkan/icd.d/* 2> /dev/null
ls -d /etc/xdg/vulkan/icd.d 2> /dev/null
echo ""
else
echo "Vulkan loader not found"
echo ""
fi
) | $GZIP_CMD >> $LOG_FILENAME
# append Vulkan SC ICD info
(
echo "____________________________________________"
echo ""
vkscinfo=`ldconfig -N -v -p 2> /dev/null | /bin/grep libvulkansc.so.1 | awk 'NF>1{print $NF}'`
if [ $? -eq 0 -a -n "$vkscinfo" ]; then
echo "Found Vulkan SC loader(s):"
readlink -f ${vkscinfo} 2> /dev/null
echo ""
# See https://github.com/KhronosGroup/VulkanSC-Loader/blob/main/docs/LoaderDriverInterface.md
echo "Listing common ICD paths:"
ls -d /usr/local/etc/vulkansc/icd.d/* 2> /dev/null
ls -d /usr/local/share/vulkansc/icd.d/* 2> /dev/null
ls -d /etc/vulkansc/icd.d/* 2> /dev/null
ls -d /usr/share/vulkansc/icd.d/* 2> /dev/null
ls -d /etc/xdg/vulkansc/icd.d 2> /dev/null
echo ""
else
echo "Vulkan SC loader not found"
echo ""
fi
) | $GZIP_CMD >> $LOG_FILENAME
# lspci information
(
echo "____________________________________________"
echo ""
lspci=`which lspci 2> /dev/null | head -n 1`
if [ $? -eq 0 -a -x "$lspci" ]; then
# Capture all devices in tree format along with vendor:device IDs
echo "$lspci -nntv"
echo ""
$lspci -nntv 2> /dev/null
echo ""
echo "____________________________________________"
echo ""
# Capture class names and class ID along with vendor:device IDs
echo "$lspci -nn"
echo ""
$lspci -nn 2> /dev/null
echo ""
echo "____________________________________________"
echo ""
# Capture verbose information for all devices, along with hex
# dump of whole configuration space
echo "$lspci -nnDvvvxxxx"
echo ""
$lspci -nnDvvvxxxx 2> /dev/null
else
hexdump=`which hexdump 2> /dev/null | head -n 1`
if [ $? -eq 0 -a -x "$hexdump" ]; then
# Capture hex dump of whole configuration space using sysfs file
echo "PCI devices configuration space dump using sysfs"
echo ""
# Define the base path for PCI devices
PCI_BASE_PATH="/sys/bus/pci/devices"
# Loop through each PCI device in the directory
for PCI_BDF in "$PCI_BASE_PATH"/*; do
# Extract device ID (BDF format)
PCI_ID=$(basename "$PCI_BDF")
# Define the config file path
PCI_CONFIG="$PCI_BDF/config"
# Check if the config file exists
if [ ! -f "$PCI_CONFIG" ]; then
continue
fi
# Retrieve PCI device information from sysfs and
# remove "0x" prefix if present
VENDOR_ID=$(sed s/^0x// "$PCI_BDF"/vendor)
DEVICE_ID=$(sed s/^0x// "$PCI_BDF"/device)
REVISION_ID=$(sed s/^0x// "$PCI_BDF"/revision)
CLASS_ID=$(sed s/^0x// "$PCI_BDF"/class)
# Print PCI device information (mimicking `lspci`)
echo
echo "$PCI_ID $CLASS_ID: Vendor $VENDOR_ID Device $DEVICE_ID (rev $REVISION_ID)"
# Read the config file using 'hexdump' and format output
$hexdump -v -e '16/1 "%02x " "\n"' "$PCI_CONFIG" | awk '{printf "%02x: %s\n",NR*16-16, $0}'
done
else
report_skip "lspci output" "lspci or hexdump not found"
echo ""
fi
fi
) | $GZIP_CMD >> $LOG_FILENAME
# THP (transparent huge pages) information
append /sys/kernel/mm/transparent_hugepage/enabled
append /proc/sys/vm/compaction_proactiveness
# NUMA information
(
echo "____________________________________________"
echo ""
numactl=`which numactl 2> /dev/null | head -n 1`
if [ $? -eq 0 -a -x "$numactl" ]; then
# Get hardware NUMA configuration
echo "$numactl -H"
echo ""
$numactl -H
fi
# Get autonuma information:
report_file /proc/sys/kernel/numa_balancing
# Get additional NUMA information about GPUs:
filelist="/sys/devices/system/node/has_cpu \
/sys/devices/system/node/has_memory \
/sys/devices/system/node/has_normal_memory \
/sys/devices/system/node/online \
/sys/devices/system/node/possible"
# Get GPU NUMA information
lspci=`which lspci 2> /dev/null | head -n 1`
if [ $? -eq 0 -a -x "$lspci" ]; then
gpus=`$lspci -d "10de:*" -s ".0" | awk '{print $1}'`
for gpu in $gpus; do
filelist="$filelist \
/sys/bus/pci/devices/*$gpu/local_cpulist \
/sys/bus/pci/devices/*$gpu/numa_node"
done
fi
for file in $filelist; do
report_file "$file"
done
) | $GZIP_CMD >> $LOG_FILENAME
# lsusb information
(
echo "____________________________________________"
echo ""
lsusb=`which lsusb 2> /dev/null | head -n 1`
if [ $? -eq 0 -a -x "$lsusb" ]; then
echo "$lsusb"
echo ""
$lsusb 2> /dev/null
echo ""
else
report_skip "lsusb output" "lsusb not found"
echo ""
fi
) | $GZIP_CMD >> $LOG_FILENAME
# dmidecode
(
echo "____________________________________________"
echo ""
dmidecode=`which dmidecode 2> /dev/null | head -n 1`
if [ $? -eq 0 -a -x "$dmidecode" ]; then
echo "$dmidecode"
echo ""
$dmidecode 2> /dev/null
echo ""
else
report_skip "dmidecode output" "dmidecode not found"
echo ""
fi
) | $GZIP_CMD >> $LOG_FILENAME
# module version magic
(
echo "____________________________________________"
echo ""
modinfo=`which modinfo 2> /dev/null | head -n 1`
if [ $? -eq 0 -a -x "$modinfo" ]; then
for name in $module_names; do
echo "$modinfo $name | grep vermagic"
echo ""
( $modinfo $name | grep vermagic ) 2> /dev/null
echo ""
done
else
report_skip "modinfo output" "modinfo not found"
echo ""
fi
) | $GZIP_CMD >> $LOG_FILENAME
# module parameter values
(
echo "____________________________________________"
echo ""
for name in $module_names; do
if [ -d "/sys/module/$name/parameters/" ]; then
grep -Hr . "/sys/module/$name/parameters/" 2> /dev/null
fi
done
) | $GZIP_CMD >> $LOG_FILENAME
# get any relevant kernel messages
(
echo "____________________________________________"
echo ""
echo "Scanning kernel log files for NVIDIA kernel messages:"
grep_args="-e NVRM -e nvidia- -e nvrm-nvlog -e nvidia-powerd"
logfound=0
search_string_in_logs /var/log/messages "$grep_args" && logfound=1
search_string_in_logs /var/log/kern.log "$grep_args" && logfound=1
search_string_in_logs /var/log/kernel.log "$grep_args" && logfound=1
search_string_in_logs /var/log/dmesg "$grep_args" && logfound=1
journalctl=`which journalctl 2> /dev/null | head -n 1`
if [ $? -eq 0 -a -x "$journalctl" ]; then
logfound=1
nvrmfound=0
for boot in -0 -1 -2; do
if (journalctl -b $boot | grep ${grep_args}) > /dev/null 2>&1; then
echo ""
echo " journalctl -b $boot:"
(journalctl -b $boot | grep ${grep_args}) 2> /dev/null
nvrmfound=1
fi
done
if [ $nvrmfound -eq 0 ]; then
echo ""
echo "No NVIDIA kernel messages found in recent systemd journal entries."
fi
fi
if [ $logfound -eq 0 ]; then
echo ""
echo "No suitable log found."
fi
echo ""
) | $GZIP_CMD >> $LOG_FILENAME
# If extra data collection is enabled, dump all active CPU backtraces to be
# picked up in dmesg
if [ $BUG_REPORT_EXTRA_SYSTEM_DATA -ne 0 ]; then
(
echo "____________________________________________"
echo ""
echo "Triggering SysRq backtrace on active CPUs (see dmesg output)"
sysrq_enabled=`cat /proc/sys/kernel/sysrq`
if [ "$sysrq_enabled" -ne "1" ]; then
echo 1 > /proc/sys/kernel/sysrq
fi
echo l > /proc/sysrq-trigger
if [ "$sysrq_enabled" -ne "1" ]; then
echo $sysrq_enabled > /proc/sys/kernel/sysrq
fi
) | $GZIP_CMD >> $LOG_FILENAME
fi
# append dmesg output
(
echo "____________________________________________"
echo ""
echo "dmesg:"
echo ""
dmesg 2> /dev/null
) | $GZIP_CMD >> $LOG_FILENAME
# print gcc & g++ version info
(
which gcc >/dev/null 2>&1
if [ $? -eq 0 ]; then
echo "____________________________________________"
echo ""
gcc -v 2>&1
fi
which g++ >/dev/null 2>&1
if [ $? -eq 0 ]; then
echo "____________________________________________"
echo ""
g++ -v 2>&1
fi
) | $GZIP_CMD >> $LOG_FILENAME
if [ $BUG_REPORT_SAFE_MODE -eq 0 ]; then
# In case of failure, if xset returns with delay, we print the
# message from check "$?" & if it returns error immediately before kill,
# we directly write the error to the log file.
(
echo "____________________________________________"
echo ""
echo "xset -q:"
echo ""
xset -q 2>&1 & sleep 1 ; kill -9 $! > /dev/null 2>&1
if [ $? -eq 0 ]; then
# The xset process is still there.
echo "xset could not connect to an X server"
fi
) | $GZIP_CMD >> $LOG_FILENAME
# In case of failure, if nvidia-settings returns with delay, we print the
# message from check "$?" & if it returns error immediately before kill,
# we directly write the error to the log file.
(
echo "____________________________________________"
echo ""
echo "nvidia-settings -q all:"
echo ""
DISPLAY= nvidia-settings -c "$DPY" -q all 2>&1 & sleep 1 ; kill -9 $! > /dev/null 2>&1
if [ $? -eq 0 ]; then
# The nvidia-settings process is still there.
echo "nvidia-settings could not connect to an X server"
fi
) | $GZIP_CMD >> $LOG_FILENAME
# In case of failure, if xrandr returns with delay, we print the
# message from check "$?" & if it returns error immediately before kill,
# we directly write the error to the log file.
(
if [ -x "`which xrandr 2>/dev/null`" ] ; then
echo "____________________________________________"
echo ""
echo "xrandr --verbose:"
echo ""
xrandr -display $DPY --verbose 2>&1 & sleep 1 ; kill -9 $! > /dev/null 2>&1
if [ $? -eq 0 ]; then
# The xrandr process is still there.
echo "xrandr could not connect to an X server"
fi
echo "____________________________________________"
echo ""
echo "xrandr --listproviders:"
echo ""
xrandr -display $DPY --listproviders 2>&1 & sleep 1 ; kill -9 $! > /dev/null 2>&1
if [ $? -eq 0 ]; then
# The xrandr process is still there.
echo "xrandr could not connect to an X server"
fi
else
report_skip "xrandr output" "xrandr not found"
fi
) | $GZIP_CMD >> $LOG_FILENAME
(
if [ -x "`which xprop 2>/dev/null`" ] ; then
echo "____________________________________________"
echo ""
echo "Running window manager properties:"
echo ""
TMP=`xprop -root _NET_SUPPORTING_WM_CHECK 2>/dev/null & sleep 1 ; kill -9 $! > /dev/null 2>&1`
WINDOW=`echo $TMP | grep -o '0x[0-9a-z]\+'`
if [ "$WINDOW" ]; then
xprop -id "$WINDOW" 2>&1 & sleep 1 ; kill -9 $! > /dev/null 2>&1
else
echo "Unable to detect window manager properties"
fi
fi
) | $GZIP_CMD >> $LOG_FILENAME
fi
sync > /dev/null 2>&1
sync > /dev/null 2>&1
# append useful /proc files
append "/proc/cmdline"
append "/proc/cpuinfo"
append "/proc/interrupts"
append "/proc/meminfo"
append "/proc/modules"
append "/proc/version"
append "/proc/pci"
append "/proc/iomem"
append "/proc/mtrr"
append "/proc/buddyinfo"
append "/proc/driver/nvidia/version"
append "/proc/driver/nvidia/params"
append "/proc/driver/nvidia/registry"
if [ $BUG_REPORT_SAFE_MODE -eq 0 ]; then
for GPU in `ls /proc/driver/nvidia/gpus/ 2> /dev/null`; do
append "/proc/driver/nvidia/gpus/$GPU/information"
done
append_glob "/proc/driver/nvidia/warnings/*"
append_glob "/proc/driver/nvidia-modeset/*"
fi
append_glob "/proc/acpi/video/*/*/info"
append "/proc/asound/cards"
append "/proc/asound/pcm"
append "/proc/asound/modules"
append "/proc/asound/devices"
append "/proc/asound/version"
append "/proc/asound/timers"
append "/proc/asound/hwdep"
for CARD in /proc/asound/card[0-9]*; do
for CODEC in $CARD/codec*; do
[ -d $CODEC ] && append_glob "$CODEC/*"
[ -f $CODEC ] && append "$CODEC"
done
for ELD in $CARD/eld*; do
[ -f $ELD ] && append "$ELD"
done
done
# List the mapping of DRM drivers to DRM device files
(
echo "____________________________________________"
echo ""
if [ -d "/sys/class/drm" ]; then
for CARD in `find -L /sys/class/drm -maxdepth 3 -path "*/device/driver" 2>/dev/null`; do
echo_metadata $CARD
done
else
echo "/sys/class/drm not present"
fi
echo ""
) | $GZIP_CMD >> $LOG_FILENAME
# List some info about DRM nodes
(
if [ -d "/sys/class/drm" ]; then
for DRM_NODE in /sys/class/drm/* ; do
DRIVERPATH="$DRM_NODE/device/device/driver"
if [ -e "$DRIVERPATH" ]; then
REALPATH=$(readlink -f "$DRIVERPATH")
case "$REALPATH" in
*/drivers/nvidia)
for DRM_SUBPATH in enabled status dpms modes; do
if [ -e "$DRM_NODE/$DRM_SUBPATH" ]; then
append "$DRM_NODE/$DRM_SUBPATH"
fi
done
if [ -e "$DRM_NODE/edid" ]; then
append_binary_file "$DRM_NODE/edid"
fi
;;
esac
fi
done
else
echo "/sys/class/drm not present"
fi
echo ""
) | $GZIP_CMD >> $LOG_FILENAME
# List the mapping of PCI devices to DRM device files, and the existence and
# permissions of the DRM device files themselves.
(
echo "____________________________________________"
echo ""
if [ -d "/dev/dri" ]; then
for FILE in /dev/dri/by-path/* /dev/dri/card* /dev/dri/renderD*; do
echo_metadata $FILE
done
else
echo "/dev/dri not present"
fi
echo ""
) | $GZIP_CMD >> $LOG_FILENAME
# nvidia-debugdump
(
echo "____________________________________________"
echo ""
nvidia_debugdump=`which nvidia-debugdump 2> /dev/null | head -n 1`
if [ $? -eq 0 -a -x "$nvidia_debugdump" ]; then
if [ $BUG_REPORT_SAFE_MODE -eq 0 ]; then
NVDD_ARGS="-D"
else
NVDD_ARGS="--ioctl --nvlogonly"
fi
base64=`which base64 2> /dev/null | head -n 1`
if [ $? -eq 0 -a -x "$base64" ]; then
# make sure what we can write to the temp file
NVDD_TEMP_FILENAME="$TMPDIR/nvidia-debugdump-temp$$.log"
touch $NVDD_TEMP_FILENAME 2> /dev/null
if [ $? -ne 0 ]; then
report_skip "nvidia-debugdump output" "Can't create temp file $NVDD_TEMP_FILENAME"
echo ""
# don't fail here, continue
else
echo "$nvidia_debugdump ${NVDD_ARGS}"
echo ""
$nvidia_debugdump ${NVDD_ARGS} -f $NVDD_TEMP_FILENAME 2> /dev/null
$base64 $NVDD_TEMP_FILENAME 2> /dev/null
echo ""
# remove the temporary file when complete
rm $NVDD_TEMP_FILENAME 2> /dev/null
fi
else
report_skip "nvidia-debugdump output" "base64 not found"
echo ""
fi
else
report_skip "nvidia-debugdump output" "nvidia-debugdump not found"
echo ""
fi
) | $GZIP_CMD >> $LOG_FILENAME
# disable these when safemode is requested
if [ $BUG_REPORT_SAFE_MODE -eq 0 ]; then
# vulkaninfo
(
echo "____________________________________________"
echo ""
vulkaninfo=`which vulkaninfo 2> /dev/null | head -n 1`
if [ $? -eq 0 -a -x "$vulkaninfo" ]; then
echo "$vulkaninfo"
echo ""
$vulkaninfo 2> /dev/null
echo ""
else
report_skip "vulkaninfo output" "vulkaninfo not found"
echo ""
fi
) | $GZIP_CMD >> $LOG_FILENAME
# nvidia-smi
NVML_LOG_FILE="nvidia-nvml-temp$$.log"
touch $NVML_LOG_FILE 2>/dev/null
if [ -w $NVML_LOG_FILE ]; then
export __NVML_DBG_FILE=${NVML_LOG_FILE} __NVML_DBG_APPEND=1 __NVML_DBG_LVL=DEBUG
fi
(
echo "____________________________________________"
echo ""
nvidia_smi=`which nvidia-smi 2> /dev/null | head -n 1`
if [ $? -eq 0 -a -x "$nvidia_smi" ]; then
report_command "$nvidia_smi --query"
report_command "$nvidia_smi --query --unit"
report_command "$nvidia_smi nvlink --errorcounters"
report_command "$nvidia_smi nvlink --remotelinkinfo"
report_command "$nvidia_smi nvlink --status"
report_command "$nvidia_smi conf-compute --query-conf-compute"
else
report_skip "nvidia-smi output" "nvidia-smi not found"
echo ""
fi
) | $GZIP_CMD >> $LOG_FILENAME
if [ -f $NVML_LOG_FILE ]; then
append_binary_file $NVML_LOG_FILE
rm -f $NVML_LOG_FILE
unset __NVML_DBG_FILE __NVML_DBG_APPEND __NVML_DBG_LVL
fi
else
(
report_skip "nvidia-smi, vulkaninfo etc. Commands" "Skipping due to --safe-mode-argument"
echo ""
) | $GZIP_CMD >> $LOG_FILENAME
fi
# InfiniBand Support
if [ $BUG_REPORT_SAFE_MODE -eq 0 ]; then
(
echo "____________________________________________"
echo ""
# Check if ibstat is available to discover devices
ibstat_cmd=`which ibstat 2> /dev/null | head -n 1`
if [ $? -eq 0 -a -x "$ibstat_cmd" ]; then
# Run ibstat without device specification (covers all devices)
report_command "$ibstat_cmd"
# List all InfiniBand devices
devices=`$ibstat_cmd -l`
# Iterate over each device
for device in $devices; do
echo "Running InfiniBand commands for device: $device"
echo ""
# ibnetdiscover
ibnetdiscover_cmd=`which ibnetdiscover 2> /dev/null | head -n 1`
if [ $? -eq 0 -a -x "$ibnetdiscover_cmd" ]; then
report_command "$ibnetdiscover_cmd -C $device"
else
report_skip "ibnetdiscover output" "ibnetdiscover not found"
echo ""
fi
# ibswitches
ibswitches_cmd=`which ibswitches 2> /dev/null | head -n 1`
if [ $? -eq 0 -a -x "$ibswitches_cmd" ]; then
report_command "$ibswitches_cmd -C $device"
else
report_skip "ibswitches output" "ibswitches not found"
echo ""
fi
# iblinkinfo
iblinkinfo_cmd=`which iblinkinfo 2> /dev/null | head -n 1`
if [ $? -eq 0 -a -x "$iblinkinfo_cmd" ]; then
report_command "$iblinkinfo_cmd -C $device"
else
report_skip "iblinkinfo output" "iblinkinfo not found"
echo ""
fi
echo ""
done
else
report_skip "ibstat output" "ibstat not found"
echo ""
fi
) | $GZIP_CMD >> $LOG_FILENAME
else
(
report_skip "Infiniband Commands" "Skipping due to --safe-mode-argument"
echo ""
) | $GZIP_CMD >> $LOG_FILENAME
fi
# copy fabric manager (for NVSwitch based systems) default log file
append_silent "/var/log/fabricmanager.log"
# get fabric manager service status information
systemctl=`which systemctl 2> /dev/null | head -n 1`
if [ $? -eq 0 -a -x "$systemctl" ]; then
cmd="$systemctl status nvidia-fabricmanager.service"
(
echo "____________________________________________"
echo ""
echo "$cmd"
$cmd
echo ""
) 2> /dev/null | $GZIP_CMD >> $LOG_FILENAME
fi
#copy IMEX daemon (for multi-node compute systems) default log file
append_silent "/var/log/nvidia-imex.log"
append_silent "/var/log/nvidia-imex-verbose.log"
append_silent "/var/log/nvidia-imex-stats.log"
# get nvidia-imex service status information
if [ $? -eq 0 -a -x "$systemctl" ]; then
cmd="$systemctl status nvidia-imex.service"
(
echo "____________________________________________"
echo ""
echo "$cmd"
$cmd
echo ""
) 2> /dev/null | $GZIP_CMD >> $LOG_FILENAME
fi
# get nvidia-imex-ctl output
imexctl=`which nvidia-imex-ctl 2> /dev/null | head -n 1`
if [ $? -eq 0 -a -x "$imexctl" ]; then
cmd="$imexctl -N"
(
echo "____________________________________________"
echo ""
echo "$cmd"
$cmd
echo ""
) 2> /dev/null | $GZIP_CMD >> $LOG_FILENAME
fi
# Print information about the libglvnd libraries
(
if [ -e $GLVND_HELPER_BASE_PATH/glvnd_check ] ; then
echo "____________________________________________"
echo ""
echo Checking libglvnd library libraries.
print_libglvnd_library $GLVND_HELPER_BASE_PATH glx libGL.so.1
print_libglvnd_library $GLVND_HELPER_BASE_PATH glx libGLX.so.0
print_libglvnd_library $GLVND_HELPER_BASE_PATH egl libEGL.so.1
if [ "$?" -eq 0 ] ; then
print_libglvnd_library $GLVND_HELPER_BASE_PATH gl libOpenGL.so.0
print_libglvnd_library $GLVND_HELPER_BASE_PATH gl libGLESv1_CM.so.1
print_libglvnd_library $GLVND_HELPER_BASE_PATH gl libGLESv2.so.2
fi
fi
if [ -e $GLVND_HELPER_BASE_PATH/32/glvnd_check ] ; then
# We might not have a 32-bit libc available, so first check whether we
# can run the 32-bit version of glvnd_check at all.
if $GLVND_HELPER_BASE_PATH/32/glvnd_check nop 2> /dev/null ; then
echo "____________________________________________"
echo ""
echo Checking 32-bit libglvnd libraries.
print_libglvnd_library $GLVND_HELPER_BASE_PATH/32 glx libGL.so.1
print_libglvnd_library $GLVND_HELPER_BASE_PATH/32 glx libGLX.so.0
print_libglvnd_library $GLVND_HELPER_BASE_PATH/32 egl libEGL.so.1
if [ "$?" -eq 0 ] ; then
print_libglvnd_library $GLVND_HELPER_BASE_PATH/32 gl libOpenGL.so.0
print_libglvnd_library $GLVND_HELPER_BASE_PATH/32 gl libGLESv1_CM.so.1
print_libglvnd_library $GLVND_HELPER_BASE_PATH/32 gl libGLESv2.so.2
fi
else
echo "____________________________________________"
echo No 32-bit loader available, not checking 32-bit libglvnd libraries.
fi
fi
) | $GZIP_CMD >> $LOG_FILENAME
(
echo "____________________________________________"
echo ""
acpidump=`which acpidump 2> /dev/null | head -n 1`
if [ $? -eq 0 -a -x "$acpidump" ]; then
base64=`which base64 2> /dev/null | head -n 1`
if [ $? -eq 0 -a -x "$base64" ]; then
TEMP_FILENAME="$TMPDIR/acpidump-temp$$.log"
echo "$acpidump -o"
echo ""
$acpidump -o $TEMP_FILENAME 2> /dev/null
# make sure if data file is created
if [ -f "$TEMP_FILENAME" ]; then
$base64 $TEMP_FILENAME 2> /dev/null
echo ""
# remove the temporary file when complete
rm $TEMP_FILENAME 2> /dev/null
else
report_skip "acpidump output" "Can't create data file $TEMP_FILENAME"
echo ""
# don't fail here, continue
fi
else
report_skip "acpidump output" "base64 not found"
echo ""
fi
else
report_skip "acpidump output" "acpidump not found"
echo ""
fi
) | $GZIP_CMD >> $LOG_FILENAME
dump_mlx() {
MLX_CSV="$TMPDIR/mlx$$.csv"
MLX_INFO="$TMPDIR/mlx$$.info"
# mlxlink --amber_collect file will dump information to the specified csv file, and
# print information to stdout (in color).
# Capture both and remove the color escape characters.
echo "$1 -d $2 --amber_collect $MLX_CSV > $MLX_INFO 2>&1"
mlx_output=`"$1" -d "$2" --amber_collect "$MLX_CSV" > "$MLX_INFO" 2>&1`
exit_code=$?
if [ $exit_code -eq 0 ]; then
cat "$MLX_INFO" | sed -Ee 's/\x1b\[[0-9;]+m//g' 2> /dev/null
cat "$MLX_CSV" 2> /dev/null
echo "____________________________________________"
echo ""
rm -f "$MLX_INFO" 2> /dev/null
rm -f "$MLX_CSV" 2> /dev/null
break
else
echo "failed with error $exit_code retrying..."
echo "$mlx_output"
cat "$MLX_INFO" | sed -Ee 's/\x1b\[[0-9;]+m//g' 2> /dev/null
rm -f "$MLX_INFO" 2> /dev/null
rm -f "$MLX_CSV" 2> /dev/null
echo "____________________________________________"
fi
}
(
mst=`which mst 2> /dev/null | head -n 1`
if [ $? -eq 0 -a -x "$mst" ]; then
mlxlink=`which mlxlink 2> /dev/null | head -n 1`
if [ $? -eq 0 -a -x "$mlxlink" ]; then
# loads PCI driver & creates PCI devices file descriptors under /dev/mst/.
echo "____________________________________________"
echo ""
report_command "timeout 60 $mst start"
echo "____________________________________________"
echo ""
# scans and adds gpu devices
report_command "timeout 60 $mst gpu add"
echo "____________________________________________"
echo ""
# Discover and add IB devices dynamically
ib_devices=$(discover_ib_devices)
if [ -n "$ib_devices" ]; then
for ib_device in $ib_devices; do
report_command "timeout 60 $mst ib add $ib_device"
done
else
# Fallback to mlx5_0 if no devices discovered
report_command "timeout 60 $mst ib add mlx5_0"
fi
echo "____________________________________________"
echo ""
resourcedump=`which resourcedump 2> /dev/null | head -n 1`
mstdump=`which mstdump 2> /dev/null | head -n 1`
mlxreg=`which mlxreg 2> /dev/null | head -n 1`
# dumps all the scanned devices
report_command "timeout 60 $mst status -v"
echo "____________________________________________"
echo ""
# Iterate and get dumps for each of the supported devices
"$mst" status -v |
{
# Get IB devices for use in subshell
ib_devices_shell="$ib_devices"
while read tmp_line
do
echo $tmp_line | grep "netir" > /dev/null 2>&1
if [ $? -eq 0 ]; then
devName=`echo $tmp_line | awk -F" " 'NR == 1 {print $1}';`
if [ ! -z "$devName" ]; then
echo "Starting GPU MST dump..."$tmp_line
echo "____________________________________________"
dump_mlx "$mlxlink" "$devName"
if [ ! -z "$resourcedump" ]; then
report_command "$resourcedump dump -d $devName -s 0x5024"
echo "____________________________________________"
echo ""
fi
fi
fi
# Check for Quantum devices on any discovered IB device
for ib_device in $ib_devices_shell; do
echo $tmp_line | grep "Quantum" | grep "$ib_device" > /dev/null 2>&1
if [ $? -eq 0 ]; then
devName=`echo $tmp_line | awk -F" " 'NR == 1 {print $1}';`
if [ ! -z "$devName" ]; then
echo "Starting Quantum dump..."$tmp_line
echo "____________________________________________"
dump_mlx "$mlxlink" "$devName"
if [ ! -z "$resourcedump" ]; then
report_command "$resourcedump dump -d $devName -s 0x5024"
echo "____________________________________________"
echo "Starting Quantum flash dump..."
echo "____________________________________________"
report_command "$resourcedump dump -d $devName -s 0x5028"
echo "____________________________________________"
if [ ! -z "$mlxreg" ]; then
echo "Clearing Quantum flash dump..."
echo "____________________________________________"
report_command "$mlxreg -d $devName --set erase_log=0x1 --reg_name MOFDE -y"
echo "____________________________________________"
fi
echo ""
fi
fi
break
fi
done
echo $tmp_line | grep "ConnectX" > /dev/null 2>&1
if [ $? -eq 0 ]; then
devName=`echo $tmp_line | awk -F" " 'NR == 1 {print $3}';`
if [ ! -z "$devName" ]; then
# Check if this is a primary function (.0) or VF (.1, .2, etc.)
if echo "$devName" | grep -q "\.0$"; then
echo "Starting ConnectX dump (PRIMARY FUNCTION)($devName)..."$tmp_line
echo "____________________________________________"
dump_mlx "$mlxlink" "$devName"
if [ ! -z "$mstdump" ]; then
report_command "$mstdump $devName"
echo "____________________________________________"
echo ""
fi
else
echo "Skipping ConnectX VF (skipping mstdump)($devName)..."$tmp_line
fi
fi
fi
done
}
# stop due to resource contention.
echo ""
report_command "$mst stop"
echo "____________________________________________"
else
echo "____________________________________________"
echo ""
report_skip "mlxlink output" "mlxlink not found"
echo ""
fi
else
echo "____________________________________________"
echo ""
report_skip "mst output" "mst not found"
echo ""
fi
) | $GZIP_CMD >> $LOG_FILENAME
# Collect NVLSM dumps
(
NVLSM_SBIN_PATH=/opt/nvidia/nvlsm/sbin
echo "____________________________________________"
echo ""
nvlsm_bug_report=`PATH=$PATH:$NVLSM_SBIN_PATH which nvlsm-bug-report.sh 2> /dev/null | head -n 1`
if [ $? -eq 0 -a -x "${nvlsm_bug_report}" ]; then
echo "Collecting NVLink Subnet Manager (NVLSM) information..."
report_command "${nvlsm_bug_report} --stdout"
else
report_skip "nvlsm-bug-report.sh output" "nvlsm-bug-report.sh not found"
echo ""
fi
) | $GZIP_CMD >> $LOG_FILENAME
echo " complete."
echo ""
TMP_LOG=$(mktemp)
{
echo ""
echo "Summary of Skipped Sections:"
echo ""
print_skips_table
echo ""
echo "Summary of Errors:"
echo ""
print_errors_table
echo ""
} | tee "$TMP_LOG"
cat "$TMP_LOG" | $GZIP_CMD >> "$LOG_FILENAME"
rm -f "$TMP_LOG"
(
echo "____________________________________________"
# print epilogue to log file
echo ""
echo "End of NVIDIA bug report log file."
) | $GZIP_CMD >> $LOG_FILENAME
rm -f "$SKIP_TMP" "$ERROR_TMP" "$TMP_LOG"