Add disk full incident response toolkit

2026-05-05 21:44:08 +00:00
parent 5dd8c34952
commit 76e24796bb
10 changed files with 742 additions and 0 deletions
@@ -0,0 +1,78 @@
+#!/usr/bin/env bash
+set -o errexit
+set -o nounset
+set -o pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=00_env.sh
+. "$SCRIPT_DIR/00_env.sh"
+
+EXECUTE=false
+TRUNCATE_FILE=""
+RESTART_SERVICE=""
+
+usage() {
+  printf 'Usage: %s [--truncate-file <path>] [--restart-service <name>] [--execute]\n' "$(basename "$0")"
+}
+
+while [[ "$#" -gt 0 ]]; do
+  case "$1" in
+    --truncate-file) TRUNCATE_FILE="${2:-}"; shift 2 ;;
+    --restart-service) RESTART_SERVICE="${2:-}"; shift 2 ;;
+    --execute) EXECUTE=true; DRY_RUN=false; shift ;;
+    -h|--help) usage; exit 0 ;;
+    *) critical "Unknown argument: $1"; usage; exit 2 ;;
+  esac
+done
+
+section "Emergency Disk Full Quick Fix Options"
+cat <<OPTIONS | tee -a "$LOG_FILE"
+Possible actions after incident commander approval:
+1. Truncate a verified active log file:
+   $0 --truncate-file /path/to/large.log --execute
+
+2. Restart a specific service holding deleted files open:
+   $0 --restart-service service-name --execute
+
+Review application impact before either action. Truncation preserves the file inode but destroys file contents.
+OPTIONS
+
+if [[ -z "$TRUNCATE_FILE" && -z "$RESTART_SERVICE" ]]; then
+  ok "No quick fix requested. Printed options only."
+  exit 0
+fi
+
+if [[ "$EXECUTE" != "true" ]]; then
+  warning "Quick fix arguments supplied without --execute. No changes made."
+  exit 0
+fi
+
+confirm_execute "emergency disk-full quick fix"
+
+if [[ -n "$TRUNCATE_FILE" ]]; then
+  validate_path "$TRUNCATE_FILE" || exit 2
+
+  if [[ ! -f "$TRUNCATE_FILE" || -L "$TRUNCATE_FILE" ]]; then
+    critical "Refusing to truncate non-regular file or symlink: $TRUNCATE_FILE"
+    exit 2
+  fi
+
+  warning "Truncating file contents: $TRUNCATE_FILE"
+  : > "$TRUNCATE_FILE"
+  ok "Truncated $TRUNCATE_FILE"
+fi
+
+if [[ -n "$RESTART_SERVICE" ]]; then
+  if [[ "$RESTART_SERVICE" == *"/"* || "$RESTART_SERVICE" == *".."* ]]; then
+    critical "Invalid service name: $RESTART_SERVICE"
+    exit 2
+  fi
+
+  if require_cmd systemctl; then
+    run_cmd systemctl restart "$RESTART_SERVICE"
+    ok "Restart requested for service: $RESTART_SERVICE"
+  else
+    critical "systemctl is required to restart services"
+    exit 1
+  fi
+fi