#!/usr/bin/env bash
# DGX Spark acceptance tools. Install only; no stress workload is started.
set -Eeuo pipefail
if [[ $(uname -s) != Linux ]]; then echo 'E_SYSTEM：请在 DGX Spark 的终端执行此命令。'; exit 1; fi
if [[ $EUID != 0 ]]; then
  echo '请输这台机器的登录密码以安装工具（输入时不显示字符）。'
  exec sudo bash "$0" "$@"
fi
export DEBIAN_FRONTEND=noninteractive
LOG=/var/log/spark-tools-install.log
exec > >(tee -a "$LOG") 2>&1
fail() { echo; echo "$1"; echo "完整日志：$LOG；修复后重跑同一条命令即可。"; exit 1; }
trap 'fail "E_INSTALL：第 ${LINENO} 行执行失败，请把日志发给远程操作人员。"' ERR
[[ -r /etc/os-release ]] || fail 'E_SYSTEM：无法识别系统。'
. /etc/os-release
[[ ${ID:-} == ubuntu && ${VERSION_ID%%.*} -ge 24 ]] || fail 'E_SYSTEM：此脚本适用于 Ubuntu 24.04 及以上的 DGX 系统。'
exec 9>/var/lock/spark-tools-install.lock
flock -n 9 || fail 'E_BUSY：已有一个工具安装进程，请等待它结束。'
WORK=$(mktemp -d /var/tmp/spark-tools.XXXXXX)
trap 'rm -rf "$WORK"' EXIT
printf '\n[1/4] 安装 SSD、CPU、内存检查工具及编译依赖。\n'
APT_SCOPE=()
if [[ -f /etc/apt/sources.list.d/ubuntu.sources ]]; then
  APT_SCOPE=(-o Dir::Etc::sourcelist=/etc/apt/sources.list.d/ubuntu.sources -o Dir::Etc::sourceparts=-)
  echo '本次只使用系统 Ubuntu 软件源下载检查工具，避免其他软件源异常影响安装。'
fi
heartbeat() {
  "$@" & local job=$!
  while kill -0 "$job" 2>/dev/null; do
    sleep 5
    if kill -0 "$job" 2>/dev/null; then echo '  仍在执行，请等待；下载与安装详情见上方输出。'; fi
  done
  local result=0
  wait "$job" || result=$?
  return "$result"
}
apt_step() {
  local attempt
  for attempt in 1 2 3; do
    echo "软件源操作，第 $attempt/3 次尝试。"
    local -a runner=()
    if [[ $attempt == 2 ]] && command -v spark-proxy >/dev/null; then
      echo '尝试通过已安装的代理下载。'
      runner=(spark-proxy)
    fi
    if heartbeat timeout --signal=TERM --kill-after=15 600 "${runner[@]}" apt-get "${APT_SCOPE[@]}" \
      -o Acquire::Retries=3 -o Acquire::http::Timeout=20 -o Acquire::https::Timeout=20 \
      -o DPkg::Lock::Timeout=90 -o Dpkg::Use-Pty=0 "$@"; then return 0; fi
    echo '本次没有完成，5 秒后自动重试。'; sleep 5
  done
  fail 'E_APT：软件包下载或安装失败。请确认联网，再运行 spark-proxy --check 检查代理；不要删除系统锁文件。'
}
apt_step update -o APT::Update::Error-Mode=any
apt_step install -y --no-install-recommends smartmontools stress-ng memtester build-essential curl ca-certificates
printf '\n[2/4] 查找机器自带的 CUDA 编译器。\n'
NVCC=$(command -v nvcc || true)
if [[ -z $NVCC ]]; then
  for candidate in /usr/local/cuda/bin/nvcc /usr/local/cuda-*/bin/nvcc; do
    if [[ -x $candidate ]]; then NVCC=$candidate; break; fi
  done
fi
[[ -n $NVCC ]] || fail 'E_CUDA：SSD、CPU、内存工具已安装，但未找到 nvcc。请把此日志发来，确认机器预装的 CUDA Toolkit 后再补齐 GPU 工具。'
"$NVCC" --version
GPU_CODES=$("$NVCC" --list-gpu-code)
if [[ $GPU_CODES == *sm_121* ]]; then ARCH=121
elif [[ $GPU_CODES == *sm_120* ]]; then ARCH=120
else fail 'E_CUDA_VERSION：当前 CUDA 编译器不支持 Blackwell。请先由远程操作人员确认 DGX 系统和 CUDA 版本。'; fi
printf '\n[3/4] 下载 NVIDIA 官方检查程序源码（约 24 KB）。\n'
SHA=9185bda888338d1774cb7925e8668844f3d755268355dadff7c800ac9d30f05d
FETCHED=0
for base in https://spark.sysu-sam.com https://cloud.20020606.xyz/spark-bootstrap http://104.168.255.41:18089 https://monitor.43-134-78-71.sslip.io/spark-bootstrap; do
  for mode in direct proxy; do
    runner=()
    if [[ $mode == proxy ]]; then
      command -v spark-proxy >/dev/null || continue
      runner=(spark-proxy)
    fi
    echo "下载节点：$base，连接方式：$mode；失败会重试并换节点。"
    if "${runner[@]}" curl -fL --retry 2 --retry-delay 2 --connect-timeout 10 --max-time 60 \
      "$base/cuda-checks-13.0.tar.gz" -o "$WORK/source.tgz"; then
      if echo "$SHA  $WORK/source.tgz" | sha256sum -c -; then FETCHED=1; break 2; fi
      echo '文件校验不通过，换节点重试。'
    fi
  done
done
[[ $FETCHED == 1 ]] || fail 'E_DOWNLOAD：所有源码下载节点均未成功，请确认网络后重跑。'
tar -xzf "$WORK/source.tgz" --no-same-owner -C "$WORK"
SRC=$WORK/cuda-samples-13.0
printf '\n[4/4] 编译 GPU 检查程序（最多使用一个编译进程）。\n'
heartbeat timeout --kill-after=15 300 "$NVCC" -O2 -std=c++17 -I "$SRC/Common" \
  "$SRC/Samples/1_Utilities/deviceQuery/deviceQuery.cpp" -o "$WORK/deviceQuery" \
  || fail 'E_COMPILE：GPU 信息程序编译失败。基础检查工具已安装，请把日志发来核对 CUDA 兼容性。'
heartbeat timeout --kill-after=15 300 "$NVCC" -O2 -std=c++17 -I "$SRC/Common" \
  -gencode "arch=compute_$ARCH,code=[sm_$ARCH,compute_$ARCH]" \
  "$SRC/Samples/0_Introduction/matrixMul/matrixMul.cu" -o "$WORK/matrixMul" \
  || fail 'E_COMPILE：GPU 运算程序编译失败，请把日志发来核对 CUDA 兼容性。'
install -d -m 755 /opt/spark-tools/bin /opt/spark-tools/licenses
install -m 755 "$WORK/deviceQuery" "$WORK/matrixMul" /opt/spark-tools/bin/
install -m 644 "$SRC/LICENSE" /opt/spark-tools/licenses/NVIDIA-CUDA-Samples-LICENSE
ln -sfn /opt/spark-tools/bin/deviceQuery /usr/local/bin/spark-device-query
ln -sfn /opt/spark-tools/bin/matrixMul /usr/local/bin/spark-matrix-mul
for tool in smartctl stress-ng memtester spark-device-query spark-matrix-mul; do
  command -v "$tool" || fail "E_VERIFY：未找到 $tool，安装未完成。"
done
printf '\n安装完成：SSD、CPU、内存和 GPU 检查工具均已准备好。\n'
echo '查看 GPU：spark-device-query'
echo '检查 GPU 计算：spark-matrix-mul'
echo '列出可读取健康信息的硬盘：sudo smartctl --scan-open'
echo 'CPU / 内存测试工具：stress-ng、memtester，等待远程人员设置测试时长和内存用量。'
echo '本次只安装工具，没有启动压力测试。'
echo "安装日志：$LOG"
