Add disk full incident response toolkit

2026-05-05 21:44:08 +00:00
parent 5dd8c34952
commit 76e24796bb
10 changed files with 742 additions and 0 deletions
@@ -0,0 +1,124 @@
+#!/usr/bin/env bash
+set -o errexit
+set -o nounset
+set -o pipefail
+
+TIMESTAMP="${TIMESTAMP:-$(date +%Y%m%d_%H%M%S)}"
+DRY_RUN="${DRY_RUN:-true}"
+LOG_FILE="${LOG_FILE:-/tmp/disk_full_${TIMESTAMP}.log}"
+WARN_THRESHOLD="${WARN_THRESHOLD:-80}"
+CRIT_THRESHOLD="${CRIT_THRESHOLD:-90}"
+EMERGENCY_THRESHOLD="${EMERGENCY_THRESHOLD:-95}"
+
+log() {
+  local level="$1"
+  shift
+  local message="$*"
+
+  printf '%s: %s\n' "$level" "$message" | tee -a "$LOG_FILE"
+}
+
+ok() {
+  log "OK" "$@"
+}
+
+warning() {
+  log "WARNING" "$@"
+}
+
+critical() {
+  log "CRITICAL" "$@"
+}
+
+section() {
+  printf '\n== %s ==\n' "$1" | tee -a "$LOG_FILE"
+}
+
+require_cmd() {
+  local cmd="$1"
+
+  if command -v "$cmd" >/dev/null 2>&1; then
+    return 0
+  fi
+
+  warning "Command not available: $cmd"
+  return 1
+}
+
+run_cmd() {
+  if [[ "$#" -eq 0 ]]; then
+    critical "run_cmd called without a command"
+    return 2
+  fi
+
+  if [[ "$DRY_RUN" == "true" ]]; then
+    ok "DRY-RUN: $*"
+    return 0
+  fi
+
+  ok "RUN: $*"
+  "$@" 2>&1 | tee -a "$LOG_FILE"
+}
+
+confirm_execute() {
+  local target="${1:-disk-full remediation}"
+
+  if [[ "$DRY_RUN" == "true" ]]; then
+    ok "Safe mode enabled. No destructive actions will be taken."
+    return 0
+  fi
+
+  warning "Execution mode requested for: $target"
+  warning "Confirm the affected filesystem, application impact, backups, and change approval before continuing."
+  printf 'Type EXECUTE to continue: '
+  read -r confirmation
+
+  if [[ "$confirmation" != "EXECUTE" ]]; then
+    critical "Confirmation failed. Aborting."
+    exit 1
+  fi
+
+  ok "Execution confirmed by operator."
+}
+
+validate_path() {
+  local path="$1"
+
+  if [[ -z "$path" ]]; then
+    critical "Path cannot be empty"
+    return 2
+  fi
+
+  if [[ ! -e "$path" ]]; then
+    critical "Path does not exist: $path"
+    return 2
+  fi
+}
+
+usage_percent_number() {
+  local value="$1"
+  printf '%s\n' "${value%\%}"
+}
+
+status_for_percent() {
+  local percent="$1"
+
+  if (( percent >= EMERGENCY_THRESHOLD )); then
+    printf 'CRITICAL'
+  elif (( percent >= CRIT_THRESHOLD )); then
+    printf 'WARNING'
+  elif (( percent >= WARN_THRESHOLD )); then
+    printf 'WARNING'
+  else
+    printf 'OK'
+  fi
+}
+
+safe_find_prune_args() {
+  printf '%s\n' \
+    -path /proc -o \
+    -path /sys -o \
+    -path /dev -o \
+    -path /run -o \
+    -path /tmp/systemd-private-\*
+}