Add AI lab maintenance toolkit

2026-06-06 00:10:44 +00:00
parent 1843796e92
commit 8cb92de06f
21 changed files with 1031 additions and 0 deletions
@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+set -o errexit
+set -o nounset
+set -o pipefail
+
+LOG_FILE="/var/log/ailab-disk-watch.log"
+threshold="${AILAB_DISK_THRESHOLD:-85}"
+
+if ((EUID != 0)); then
+  printf 'CRITICAL: this script must run as root to write %s\n' "$LOG_FILE" >&2
+  exit 2
+fi
+
+if [[ ! "$threshold" =~ ^[0-9]+$ ]] || ((threshold < 1 || threshold > 100)); then
+  printf 'CRITICAL: AILAB_DISK_THRESHOLD must be an integer from 1 to 100\n' >&2
+  exit 2
+fi
+
+exec > >(tee -a "$LOG_FILE") 2>&1
+printf '\n[%s] Disk usage check; threshold=%s%%\n' "$(date --iso-8601=seconds)" "$threshold"
+
+status=0
+while read -r filesystem _blocks _used available use_percent mountpoint; do
+  usage="${use_percent%\%}"
+
+  if [[ ! "$usage" =~ ^[0-9]+$ ]]; then
+    printf 'WARNING: unable to parse usage for %s mounted on %s\n' "$filesystem" "$mountpoint"
+    status=1
+  elif ((usage >= threshold)); then
+    printf 'WARNING: %s mounted on %s is %s used; threshold=%s%%; available=%s KB\n' \
+      "$filesystem" "$mountpoint" "$use_percent" "$threshold" "$available"
+    status=1
+  else
+    printf 'OK: %s mounted on %s is %s used\n' "$filesystem" "$mountpoint" "$use_percent"
+  fi
+done < <(df -P -x tmpfs -x devtmpfs | awk 'NR > 1 {print $1, $2, $3, $4, $5, $6}')
+
+exit "$status"