From c42d8bfb8f8f760f9a9500fd5944412d93a10882 Mon Sep 17 00:00:00 2001 From: Mateusz Suski Date: Tue, 5 May 2026 21:40:09 +0000 Subject: [PATCH] Add Veritas VxVM and VCS storage expansion toolkit --- infra-run/scripts/bash/veritas/00_env.sh | 238 ++++++++++++++++++ .../bash/veritas/01_detect_new_luns.sh | 42 ++++ .../bash/veritas/02_precheck_vcs_vxvm.sh | 94 +++++++ .../bash/veritas/03_freeze_vcs_group.sh | 31 +++ .../bash/veritas/04_init_vxvm_disks.sh | 56 +++++ .../bash/veritas/05_extend_diskgroup.sh | 70 ++++++ .../bash/veritas/06_extend_volume_fs.sh | 81 ++++++ .../bash/veritas/07_postcheck_vcs_vxvm.sh | 94 +++++++ .../bash/veritas/08_unfreeze_vcs_group.sh | 31 +++ infra-run/scripts/bash/veritas/README.md | 106 ++++++++ .../bash/veritas/veritas_extend_runbook.sh | 94 +++++++ 11 files changed, 937 insertions(+) create mode 100755 infra-run/scripts/bash/veritas/00_env.sh create mode 100755 infra-run/scripts/bash/veritas/01_detect_new_luns.sh create mode 100755 infra-run/scripts/bash/veritas/02_precheck_vcs_vxvm.sh create mode 100755 infra-run/scripts/bash/veritas/03_freeze_vcs_group.sh create mode 100755 infra-run/scripts/bash/veritas/04_init_vxvm_disks.sh create mode 100755 infra-run/scripts/bash/veritas/05_extend_diskgroup.sh create mode 100755 infra-run/scripts/bash/veritas/06_extend_volume_fs.sh create mode 100755 infra-run/scripts/bash/veritas/07_postcheck_vcs_vxvm.sh create mode 100755 infra-run/scripts/bash/veritas/08_unfreeze_vcs_group.sh create mode 100644 infra-run/scripts/bash/veritas/README.md create mode 100755 infra-run/scripts/bash/veritas/veritas_extend_runbook.sh diff --git a/infra-run/scripts/bash/veritas/00_env.sh b/infra-run/scripts/bash/veritas/00_env.sh new file mode 100755 index 0000000..5895f35 --- /dev/null +++ b/infra-run/scripts/bash/veritas/00_env.sh @@ -0,0 +1,238 @@ +#!/usr/bin/env bash +set -o errexit +set -o nounset +set -o pipefail + +DRY_RUN=true +TIMESTAMP="$(date +%Y%m%d_%H%M%S)" +LOG_FILE="${LOG_FILE:-/tmp/veritas_extend_${TIMESTAMP}.log}" + +SERVICE_GROUP="" +DISKGROUP="" +VOLUME="" +MOUNTPOINT="" +SIZE="" +DISKS="" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +log() { + local level="${1:-INFO}" + shift || true + local message="$*" + local line + + line="$(printf '%s [%s] %s' "$(date '+%Y-%m-%d %H:%M:%S')" "$level" "$message")" + printf '%s\n' "$line" + printf '%s\n' "$line" >> "$LOG_FILE" +} + +ok() { + log "OK" "$*" +} + +warning() { + log "WARNING" "$*" +} + +critical() { + log "CRITICAL" "$*" +} + +require_cmd() { + local cmd="$1" + + if ! command -v "$cmd" >/dev/null 2>&1; then + critical "required command not found: $cmd" + return 1 + fi + + return 0 +} + +run_cmd() { + local description="$1" + shift + + if (( "$#" == 0 )); then + critical "run_cmd called without a command" + return 2 + fi + + log "INFO" "$description" + log "INFO" "command: $*" + + if [[ "$DRY_RUN" == "true" ]]; then + log "INFO" "DRY-RUN: command not executed" + return 0 + fi + + "$@" 2>&1 | tee -a "$LOG_FILE" +} + +confirm_execute() { + local prompt="${1:-Type EXECUTE to continue with real changes}" + local answer="" + + if [[ "$DRY_RUN" == "true" ]]; then + ok "dry-run mode active; confirmation not required" + return 0 + fi + + warning "real execution mode requested with --execute" + warning "$prompt" + printf 'Type EXECUTE to continue: ' + read -r answer + + if [[ "$answer" != "EXECUTE" ]]; then + critical "confirmation failed; no changes made" + exit 2 + fi +} + +usage_common() { + cat <<'USAGE' +Common options: + --sg + --dg + --vol + --mount + --size <+SIZE> + --disks "disk1 disk2" + --execute + --help +USAGE +} + +parse_common_args() { + while (( "$#" > 0 )); do + case "$1" in + --sg) + if [[ -z "${2:-}" ]]; then + critical "missing value for --sg" + exit 2 + fi + SERVICE_GROUP="${2:-}" + shift 2 + ;; + --dg) + if [[ -z "${2:-}" ]]; then + critical "missing value for --dg" + exit 2 + fi + DISKGROUP="${2:-}" + shift 2 + ;; + --vol) + if [[ -z "${2:-}" ]]; then + critical "missing value for --vol" + exit 2 + fi + VOLUME="${2:-}" + shift 2 + ;; + --mount) + if [[ -z "${2:-}" ]]; then + critical "missing value for --mount" + exit 2 + fi + MOUNTPOINT="${2:-}" + shift 2 + ;; + --size) + if [[ -z "${2:-}" ]]; then + critical "missing value for --size" + exit 2 + fi + SIZE="${2:-}" + shift 2 + ;; + --disks) + if [[ -z "${2:-}" ]]; then + critical "missing value for --disks" + exit 2 + fi + DISKS="${2:-}" + shift 2 + ;; + --execute) + DRY_RUN=false + shift + ;; + --help|-h) + usage_common + exit 0 + ;; + *) + critical "unknown argument: $1" + usage_common + exit 2 + ;; + esac + done +} + +require_nonempty() { + local value="$1" + local name="$2" + + if [[ -z "$value" ]]; then + critical "missing required argument: $name" + return 1 + fi + + return 0 +} + +require_inputs() { + local failed=0 + local name + + for name in "$@"; do + case "$name" in + sg) require_nonempty "$SERVICE_GROUP" "--sg" || failed=1 ;; + dg) require_nonempty "$DISKGROUP" "--dg" || failed=1 ;; + vol) require_nonempty "$VOLUME" "--vol" || failed=1 ;; + mount) require_nonempty "$MOUNTPOINT" "--mount" || failed=1 ;; + size) require_nonempty "$SIZE" "--size" || failed=1 ;; + disks) require_nonempty "$DISKS" "--disks" || failed=1 ;; + *) critical "internal error: unknown required input '$name'"; failed=1 ;; + esac + done + + if (( failed != 0 )); then + usage_common + exit 2 + fi +} + +has_cmd() { + command -v "$1" >/dev/null 2>&1 +} + +capture_cmd() { + local description="$1" + shift + + log "INFO" "$description" + log "INFO" "command: $*" + "$@" 2>&1 | tee -a "$LOG_FILE" +} + +disk_status_line() { + local disk="$1" + vxdisk list "$disk" 2>/dev/null | awk -F': *' ' + /device:/ {device=$2} + /status:/ {status=$2} + END { + if (device != "" || status != "") { + print device "|" status + } + }' +} + +vxprint_volume_device() { + local dg="$1" + local vol="$2" + vxprint -g "$dg" -F '%device' "$vol" 2>/dev/null || true +} diff --git a/infra-run/scripts/bash/veritas/01_detect_new_luns.sh b/infra-run/scripts/bash/veritas/01_detect_new_luns.sh new file mode 100755 index 0000000..fc847ef --- /dev/null +++ b/infra-run/scripts/bash/veritas/01_detect_new_luns.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +set -o errexit +set -o nounset +set -o pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=00_env.sh +source "$SCRIPT_DIR/00_env.sh" + +parse_common_args "$@" + +missing=0 +for cmd in lsblk vxdisk vxdctl; do + require_cmd "$cmd" || missing=1 +done + +if (( missing != 0 )); then + exit 2 +fi + +ok "Veritas LUN discovery started" +log "INFO" "log file: $LOG_FILE" +capture_cmd "Current Linux block devices" lsblk +capture_cmd "Current VxVM disks" vxdisk list + +run_cmd "Refresh VxVM device discovery" vxdctl enable +run_cmd "Scan disks known to VxVM" vxdisk scandisks + +ok "Candidate VxVM disks with status 'online invalid'" +candidate_count=0 +while read -r disk status rest; do + if [[ "$status $rest" == *"online invalid"* ]]; then + printf ' %s %s %s\n' "$disk" "$status" "$rest" | tee -a "$LOG_FILE" + candidate_count=$(( candidate_count + 1 )) + fi +done < <(vxdisk list 2>/dev/null | awk 'NR > 1 {print $1, $4, $5, $6, $7, $8}') + +if (( candidate_count == 0 )); then + warning "no candidate disks detected with VxVM status 'online invalid'" +else + ok "detected $candidate_count candidate disk(s); review before initialization" +fi diff --git a/infra-run/scripts/bash/veritas/02_precheck_vcs_vxvm.sh b/infra-run/scripts/bash/veritas/02_precheck_vcs_vxvm.sh new file mode 100755 index 0000000..5ba3fca --- /dev/null +++ b/infra-run/scripts/bash/veritas/02_precheck_vcs_vxvm.sh @@ -0,0 +1,94 @@ +#!/usr/bin/env bash +set -o errexit +set -o nounset +set -o pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=00_env.sh +source "$SCRIPT_DIR/00_env.sh" + +parse_common_args "$@" +require_inputs sg dg vol mount + +missing=0 +for cmd in hastatus hagrp hares vxdisk vxdg vxprint df findmnt; do + require_cmd "$cmd" || missing=1 +done + +if (( missing != 0 )); then + exit 2 +fi + +status=0 +ok "Precheck started for service group '$SERVICE_GROUP', diskgroup '$DISKGROUP', volume '$VOLUME'" +log "INFO" "log file: $LOG_FILE" + +if hastatus -sum >/dev/null 2>&1; then + ok "VCS status is available" +else + critical "VCS does not appear to be running or hastatus failed" + status=1 +fi + +if hagrp -display "$SERVICE_GROUP" >/dev/null 2>&1; then + ok "service group exists: $SERVICE_GROUP" +else + critical "service group not found: $SERVICE_GROUP" + status=1 +fi + +group_state="$(hagrp -state "$SERVICE_GROUP" 2>/dev/null || true)" +printf '%s\n' "$group_state" | tee -a "$LOG_FILE" +if printf '%s\n' "$group_state" | grep -qi "ONLINE"; then + ok "service group is online" +else + critical "service group is not online" + status=1 +fi + +online_node="$(printf '%s\n' "$group_state" | awk '/ONLINE/ {print $NF; exit}')" +if [[ -n "$online_node" ]]; then + ok "possible online node: $online_node" +else + warning "unable to identify online node from hagrp output" +fi + +if vxdg list "$DISKGROUP" >/dev/null 2>&1; then + ok "diskgroup exists: $DISKGROUP" +else + critical "diskgroup not found: $DISKGROUP" + status=1 +fi + +if vxprint -g "$DISKGROUP" "$VOLUME" >/dev/null 2>&1; then + ok "volume exists: $VOLUME" +else + critical "volume not found in diskgroup: $VOLUME" + status=1 +fi + +if findmnt --target "$MOUNTPOINT" >/dev/null 2>&1; then + ok "mountpoint is mounted: $MOUNTPOINT" + fs_type="$(findmnt --noheadings --output FSTYPE --target "$MOUNTPOINT" | awk 'NR == 1 {print $1}')" + ok "filesystem type: ${fs_type:-unknown}" +else + critical "mountpoint is not mounted: $MOUNTPOINT" + status=1 +fi + +capture_cmd "Current filesystem usage" df -h "$MOUNTPOINT" || status=1 +capture_cmd "Current VxVM layout" vxprint -g "$DISKGROUP" -ht || status=1 +capture_cmd "Current VCS service group display" hagrp -display "$SERVICE_GROUP" || status=1 +if hares -display 2>/dev/null | grep -F "$SERVICE_GROUP" | tee -a "$LOG_FILE"; then + ok "displayed VCS resources related to service group: $SERVICE_GROUP" +else + warning "no VCS resource display rows matched service group: $SERVICE_GROUP" +fi + +if (( status == 0 )); then + ok "precheck completed successfully" +else + critical "precheck found one or more issues" +fi + +exit "$status" diff --git a/infra-run/scripts/bash/veritas/03_freeze_vcs_group.sh b/infra-run/scripts/bash/veritas/03_freeze_vcs_group.sh new file mode 100755 index 0000000..d3955af --- /dev/null +++ b/infra-run/scripts/bash/veritas/03_freeze_vcs_group.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +set -o errexit +set -o nounset +set -o pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=00_env.sh +source "$SCRIPT_DIR/00_env.sh" + +parse_common_args "$@" +require_inputs sg + +missing=0 +for cmd in hagrp grep; do + require_cmd "$cmd" || missing=1 +done + +if (( missing != 0 )); then + exit 2 +fi + +ok "Current service group state" +capture_cmd "hagrp state for $SERVICE_GROUP" hagrp -state "$SERVICE_GROUP" +warning "Freezing a VCS service group prevents automatic failover actions while the freeze is active" + +confirm_execute "This will persistently freeze VCS service group '$SERVICE_GROUP'." +run_cmd "Freeze VCS service group persistently" hagrp -freeze "$SERVICE_GROUP" -persistent + +ok "Freeze state check" +hagrp -display "$SERVICE_GROUP" 2>&1 | tee -a "$LOG_FILE" | grep -i "Frozen" || true +ok "freeze step completed" diff --git a/infra-run/scripts/bash/veritas/04_init_vxvm_disks.sh b/infra-run/scripts/bash/veritas/04_init_vxvm_disks.sh new file mode 100755 index 0000000..cbb4478 --- /dev/null +++ b/infra-run/scripts/bash/veritas/04_init_vxvm_disks.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +set -o errexit +set -o nounset +set -o pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=00_env.sh +source "$SCRIPT_DIR/00_env.sh" + +parse_common_args "$@" +require_inputs disks + +missing=0 +for cmd in vxdisk vxdisksetup awk; do + require_cmd "$cmd" || missing=1 +done + +if (( missing != 0 )); then + exit 2 +fi + +status=0 +for disk in $DISKS; do + if ! vxdisk list "$disk" >/dev/null 2>&1; then + critical "disk not found in vxdisk list: $disk" + status=1 + continue + fi + + info="$(disk_status_line "$disk")" + disk_status="${info#*|}" + if [[ "$disk_status" != *"online invalid"* ]]; then + critical "disk '$disk' is not safe to initialize; status is '${disk_status:-unknown}', expected 'online invalid'" + status=1 + continue + fi + + ok "disk '$disk' validated as online invalid" +done + +if (( status != 0 )); then + critical "one or more disks failed validation; no initialization attempted" + exit 1 +fi + +confirm_execute "This will initialize VxVM metadata on disk(s): $DISKS" + +for disk in $DISKS; do + run_cmd "Initialize VxVM disk $disk" vxdisksetup -i "$disk" +done + +for disk in $DISKS; do + capture_cmd "Post-initialization VxVM disk state for $disk" vxdisk list "$disk" +done + +ok "disk initialization step completed" diff --git a/infra-run/scripts/bash/veritas/05_extend_diskgroup.sh b/infra-run/scripts/bash/veritas/05_extend_diskgroup.sh new file mode 100755 index 0000000..c2a0531 --- /dev/null +++ b/infra-run/scripts/bash/veritas/05_extend_diskgroup.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +set -o errexit +set -o nounset +set -o pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=00_env.sh +source "$SCRIPT_DIR/00_env.sh" + +parse_common_args "$@" +require_inputs dg disks + +missing=0 +for cmd in vxdg vxdisk vxprint tr; do + require_cmd "$cmd" || missing=1 +done + +if (( missing != 0 )); then + exit 2 +fi + +if ! vxdg list "$DISKGROUP" >/dev/null 2>&1; then + critical "diskgroup not found: $DISKGROUP" + exit 1 +fi +ok "diskgroup exists: $DISKGROUP" + +status=0 +for disk in $DISKS; do + if ! vxdisk list "$disk" >/dev/null 2>&1; then + critical "disk not found in vxdisk list: $disk" + status=1 + continue + fi + + summary="$(vxdisk list 2>/dev/null | awk -v disk="$disk" '$1 == disk {print $0}')" + if [[ -z "$summary" ]]; then + warning "unable to find summary row for $disk; using detailed status only" + elif printf '%s\n' "$summary" | awk '{print $3}' | grep -qv '^-'; then + critical "disk '$disk' appears to belong to a diskgroup: $summary" + status=1 + continue + fi + + info="$(disk_status_line "$disk")" + disk_status="${info#*|}" + if [[ "$disk_status" == *"online invalid"* ]]; then + critical "disk '$disk' is still online invalid; initialize it before adding to a diskgroup" + status=1 + continue + fi + + ok "disk '$disk' appears initialized and unassigned" +done + +if (( status != 0 )); then + critical "one or more disks failed validation; diskgroup extension not attempted" + exit 1 +fi + +confirm_execute "This will add disk(s) '$DISKS' to VxVM diskgroup '$DISKGROUP'." + +for disk in $DISKS; do + alias_base="$(printf '%s_%s' "$DISKGROUP" "$disk" | tr -c 'A-Za-z0-9_' '_')" + run_cmd "Add disk $disk to diskgroup $DISKGROUP as $alias_base" vxdg -g "$DISKGROUP" adddisk "${alias_base}=${disk}" +done + +capture_cmd "Diskgroup details after extension" vxdg list "$DISKGROUP" +capture_cmd "VxVM layout after diskgroup extension" vxprint -g "$DISKGROUP" -ht +ok "diskgroup extension step completed" diff --git a/infra-run/scripts/bash/veritas/06_extend_volume_fs.sh b/infra-run/scripts/bash/veritas/06_extend_volume_fs.sh new file mode 100755 index 0000000..a7aba17 --- /dev/null +++ b/infra-run/scripts/bash/veritas/06_extend_volume_fs.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +set -o errexit +set -o nounset +set -o pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=00_env.sh +source "$SCRIPT_DIR/00_env.sh" + +parse_common_args "$@" +require_inputs dg vol mount size + +missing=0 +for cmd in vxdg vxprint vxassist df findmnt; do + require_cmd "$cmd" || missing=1 +done + +if (( missing != 0 )); then + exit 2 +fi + +if [[ ! "$SIZE" =~ ^\+[0-9]+[KkMmGgTtPp]?$ ]]; then + critical "invalid --size '$SIZE'; use a grow-by value such as +10G" + exit 2 +fi + +status=0 +vxdg list "$DISKGROUP" >/dev/null 2>&1 || { critical "diskgroup not found: $DISKGROUP"; status=1; } +vxprint -g "$DISKGROUP" "$VOLUME" >/dev/null 2>&1 || { critical "volume not found: $VOLUME"; status=1; } +findmnt --target "$MOUNTPOINT" >/dev/null 2>&1 || { critical "mountpoint is not mounted: $MOUNTPOINT"; status=1; } + +if (( status != 0 )); then + exit 1 +fi + +fs_type="$(findmnt --noheadings --output FSTYPE --target "$MOUNTPOINT" | awk 'NR == 1 {print $1}')" +device="$(findmnt --noheadings --output SOURCE --target "$MOUNTPOINT" | awk 'NR == 1 {print $1}')" +ok "filesystem type: ${fs_type:-unknown}" +ok "mounted device: ${device:-unknown}" + +capture_cmd "Filesystem usage before expansion" df -h "$MOUNTPOINT" +capture_cmd "VxVM layout before volume expansion" vxprint -g "$DISKGROUP" -ht + +confirm_execute "This will grow VxVM volume '$VOLUME' in diskgroup '$DISKGROUP' by '$SIZE'." +run_cmd "Grow VxVM volume by requested size" vxassist -g "$DISKGROUP" growby "$VOLUME" "$SIZE" + +case "$fs_type" in + vxfs) + warning "VxFS fsadm syntax can vary by Veritas release and site standard" + warning "manual filesystem resize recommended after volume growth; review a command such as: fsadm -F vxfs -b $MOUNTPOINT" + ;; + xfs) + if has_cmd xfs_growfs; then + run_cmd "Resize XFS filesystem online" xfs_growfs "$MOUNTPOINT" + else + critical "xfs_growfs not found; cannot resize XFS safely" + exit 1 + fi + ;; + ext3|ext4) + if has_cmd resize2fs; then + if [[ -n "$device" ]]; then + run_cmd "Resize ext filesystem" resize2fs "$device" + else + critical "unable to detect mounted device for resize2fs" + exit 1 + fi + else + critical "resize2fs not found; cannot resize ext filesystem safely" + exit 1 + fi + ;; + *) + warning "unsupported or unknown filesystem type '$fs_type'; volume growth command was handled according to dry-run/execute mode" + warning "manual filesystem resize required after confirming platform-specific procedure" + ;; +esac + +capture_cmd "Filesystem usage after expansion attempt" df -h "$MOUNTPOINT" +capture_cmd "VxVM layout after volume expansion attempt" vxprint -g "$DISKGROUP" -ht +ok "volume and filesystem expansion step completed" diff --git a/infra-run/scripts/bash/veritas/07_postcheck_vcs_vxvm.sh b/infra-run/scripts/bash/veritas/07_postcheck_vcs_vxvm.sh new file mode 100755 index 0000000..5e492d5 --- /dev/null +++ b/infra-run/scripts/bash/veritas/07_postcheck_vcs_vxvm.sh @@ -0,0 +1,94 @@ +#!/usr/bin/env bash +set -o errexit +set -o nounset +set -o pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=00_env.sh +source "$SCRIPT_DIR/00_env.sh" + +parse_common_args "$@" +require_inputs sg dg vol mount + +missing=0 +for cmd in hagrp vxdisk vxdg vxprint df findmnt; do + require_cmd "$cmd" || missing=1 +done + +if (( missing != 0 )); then + exit 2 +fi + +status=0 +ok "Post-check started" +log "INFO" "log file: $LOG_FILE" + +group_state="$(hagrp -state "$SERVICE_GROUP" 2>/dev/null || true)" +printf '%s\n' "$group_state" | tee -a "$LOG_FILE" +if printf '%s\n' "$group_state" | grep -qi "ONLINE"; then + ok "service group is online" +else + critical "service group is not online" + status=1 +fi + +freeze_display="$(hagrp -display "$SERVICE_GROUP" 2>/dev/null | grep -i "Frozen" || true)" +printf '%s\n' "$freeze_display" | tee -a "$LOG_FILE" +if printf '%s\n' "$freeze_display" | grep -Eqi "(1|true|yes|persistent)"; then + ok "service group still appears frozen before unfreeze" +else + warning "unable to confirm service group freeze state; review before unfreezing" +fi + +if vxdg list "$DISKGROUP" >/dev/null 2>&1; then + ok "diskgroup imported and visible: $DISKGROUP" +else + critical "diskgroup not visible: $DISKGROUP" + status=1 +fi + +volume_line="$(vxprint -g "$DISKGROUP" -v "$VOLUME" 2>/dev/null || true)" +printf '%s\n' "$volume_line" | tee -a "$LOG_FILE" +if printf '%s\n' "$volume_line" | grep -Eqi "(ENABLED|ACTIVE|started|fsgen)"; then + ok "volume appears enabled or active" +else + critical "unable to confirm volume is enabled or active" + status=1 +fi + +if findmnt --target "$MOUNTPOINT" >/dev/null 2>&1; then + ok "mountpoint is mounted: $MOUNTPOINT" +else + critical "mountpoint is not mounted: $MOUNTPOINT" + status=1 +fi + +capture_cmd "Filesystem usage after expansion" df -h "$MOUNTPOINT" || status=1 +capture_cmd "VxVM layout after expansion" vxprint -g "$DISKGROUP" -ht || status=1 +capture_cmd "VxVM disk list after expansion" vxdisk list || status=1 + +if has_cmd journalctl; then + capture_cmd "Recent kernel journal messages" journalctl -k -n 50 || warning "journalctl check failed; review permissions or system logging" +else + warning "journalctl not found; skipping kernel journal check" +fi + +if has_cmd dmesg; then + log "INFO" "Recent dmesg messages" + log "INFO" "command: dmesg -T | tail -50" + if dmesg -T 2>&1 | tail -50 | tee -a "$LOG_FILE"; then + ok "captured recent dmesg messages" + else + warning "dmesg check failed; review permissions or kernel logging" + fi +else + warning "dmesg not found; skipping dmesg check" +fi + +if (( status == 0 )); then + ok "post-check completed successfully; compare df output with precheck baseline for expected size increase" +else + critical "post-check found one or more issues" +fi + +exit "$status" diff --git a/infra-run/scripts/bash/veritas/08_unfreeze_vcs_group.sh b/infra-run/scripts/bash/veritas/08_unfreeze_vcs_group.sh new file mode 100755 index 0000000..436c361 --- /dev/null +++ b/infra-run/scripts/bash/veritas/08_unfreeze_vcs_group.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +set -o errexit +set -o nounset +set -o pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=00_env.sh +source "$SCRIPT_DIR/00_env.sh" + +parse_common_args "$@" +require_inputs sg + +missing=0 +for cmd in hagrp grep; do + require_cmd "$cmd" || missing=1 +done + +if (( missing != 0 )); then + exit 2 +fi + +ok "Current service group freeze state" +hagrp -display "$SERVICE_GROUP" 2>&1 | tee -a "$LOG_FILE" | grep -i "Frozen" || true + +confirm_execute "This will persistently unfreeze VCS service group '$SERVICE_GROUP'." +run_cmd "Unfreeze VCS service group persistently" hagrp -unfreeze "$SERVICE_GROUP" -persistent + +ok "Verify service group freeze state" +hagrp -display "$SERVICE_GROUP" 2>&1 | tee -a "$LOG_FILE" | grep -i "Frozen" || true +capture_cmd "Current service group state" hagrp -state "$SERVICE_GROUP" +ok "unfreeze step completed" diff --git a/infra-run/scripts/bash/veritas/README.md b/infra-run/scripts/bash/veritas/README.md new file mode 100644 index 0000000..4c73e12 --- /dev/null +++ b/infra-run/scripts/bash/veritas/README.md @@ -0,0 +1,106 @@ +# Veritas VxVM/VCS Storage Expansion Toolkit + +Production-style Bash examples for expanding storage in a Veritas environment. These scripts are sanitized operational tooling for a Linux Infrastructure Engineer portfolio: they show the flow, guardrails, logging, and validation patterns used in enterprise change work. + +## VxVM vs VCS + +Veritas Volume Manager (VxVM) manages disks, disk groups, volumes, plexes, and subdisks. It is the storage virtualization layer used to initialize SAN LUNs, add capacity to disk groups, and grow volumes. + +Veritas Cluster Server (VCS) manages application availability through service groups and resources. During storage changes, freezing the relevant service group can prevent unexpected automated failover actions while operators perform controlled work. + +## Safety Notes + +- Default mode is always dry-run. +- Real execution requires `--execute`. +- Mutating scripts require an interactive `EXECUTE` confirmation after `--execute`. +- Disk names are never assumed. Candidate disks must be supplied explicitly. +- Disks are initialized only when VxVM reports the expected `online invalid` state. +- Filesystem growth is conservative and depends on detected filesystem type. +- Exact Veritas and filesystem commands can differ by product version, OS, and site standards. + +## Required Tools + +Common commands used by the toolkit: + +- Linux: `bash`, `lsblk`, `df`, `findmnt`, `awk`, `grep`, `tee` +- VCS: `hastatus`, `hagrp`, `hares` +- VxVM: `vxdctl`, `vxdisk`, `vxdisksetup`, `vxdg`, `vxprint`, `vxassist` +- Filesystem resize tools as applicable: `fsadm`, `xfs_growfs`, `resize2fs` +- Optional log checks: `journalctl`, `dmesg` + +## Scripts + +- `00_env.sh` - shared configuration, logging, dry-run handling, argument helpers. +- `01_detect_new_luns.sh` - discovers Linux block devices and VxVM `online invalid` candidates. +- `02_precheck_vcs_vxvm.sh` - validates cluster, diskgroup, volume, and filesystem state. +- `03_freeze_vcs_group.sh` - freezes a VCS service group. +- `04_init_vxvm_disks.sh` - initializes candidate VxVM disks. +- `05_extend_diskgroup.sh` - adds initialized disks to a diskgroup. +- `06_extend_volume_fs.sh` - grows a VxVM volume and resizes the filesystem where safe. +- `07_postcheck_vcs_vxvm.sh` - validates final state and gathers post-change evidence. +- `08_unfreeze_vcs_group.sh` - unfreezes the VCS service group. +- `veritas_extend_runbook.sh` - prints the recommended order and can optionally run the steps. + +## Example Workflow + +Print the runbook only: + +```bash +./veritas_extend_runbook.sh \ + --sg app_sg \ + --dg appdg \ + --vol appvol \ + --mount /app \ + --size +100G \ + --disks "emc0_1234 emc0_1235" +``` + +Run all steps in dry-run mode: + +```bash +./veritas_extend_runbook.sh \ + --run \ + --sg app_sg \ + --dg appdg \ + --vol appvol \ + --mount /app \ + --size +100G \ + --disks "emc0_1234 emc0_1235" +``` + +Run a controlled execution. Each mutating step still asks for `EXECUTE`: + +```bash +./veritas_extend_runbook.sh \ + --run \ + --execute \ + --sg app_sg \ + --dg appdg \ + --vol appvol \ + --mount /app \ + --size +100G \ + --disks "emc0_1234 emc0_1235" +``` + +Run individual steps: + +```bash +./01_detect_new_luns.sh +./02_precheck_vcs_vxvm.sh --sg app_sg --dg appdg --vol appvol --mount /app +./03_freeze_vcs_group.sh --sg app_sg +./04_init_vxvm_disks.sh --disks "emc0_1234 emc0_1235" +./05_extend_diskgroup.sh --dg appdg --disks "emc0_1234 emc0_1235" +./06_extend_volume_fs.sh --dg appdg --vol appvol --mount /app --size +100G +./07_postcheck_vcs_vxvm.sh --sg app_sg --dg appdg --vol appvol --mount /app +./08_unfreeze_vcs_group.sh --sg app_sg +``` + +## Exit Codes + +- `0` - OK. +- `1` - operational validation or execution failure. +- `2` - invalid input or missing required command. + +## Operational Reminder + +Use these scripts as examples and adapt them to local runbooks, naming standards, multipath stack, Veritas release, filesystem type, and change-control policy before production use. diff --git a/infra-run/scripts/bash/veritas/veritas_extend_runbook.sh b/infra-run/scripts/bash/veritas/veritas_extend_runbook.sh new file mode 100755 index 0000000..cd0a69a --- /dev/null +++ b/infra-run/scripts/bash/veritas/veritas_extend_runbook.sh @@ -0,0 +1,94 @@ +#!/usr/bin/env bash +set -o errexit +set -o nounset +set -o pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=00_env.sh +source "$SCRIPT_DIR/00_env.sh" + +RUN_STEPS=false + +usage() { + cat <<'USAGE' +Usage: + ./veritas_extend_runbook.sh --sg --dg --vol --mount --size <+SIZE> --disks "disk1 disk2" [--execute] [--run] + +Options: + --run Run each step in the recommended order. Without --run, only print the runbook. + --execute Pass --execute to change steps. Dry-run remains the default. +USAGE + usage_common +} + +args=() +while (( "$#" > 0 )); do + case "$1" in + --run) + RUN_STEPS=true + shift + ;; + --help|-h) + usage + exit 0 + ;; + *) + args+=("$1") + shift + ;; + esac +done + +parse_common_args "${args[@]}" + +cat <