Add Slurm AI/HPC cluster platform project

2026-06-04 19:41:05 +00:00
parent e2624a7533
commit d300d490f5
49 changed files with 4777 additions and 0 deletions
@@ -0,0 +1,90 @@
+---
+- name: Backup SlurmDBD MariaDB database
+  hosts: slurm_controller
+  become: true
+  gather_facts: true
+
+  vars:
+    slurmdbd_backup_dir: /var/backups/slurmdbd
+    local_fetch_dir: "{{ playbook_dir }}/../../artifacts/backups/slurmdbd"
+
+  tasks:
+    - name: Create remote backup directory
+      ansible.builtin.file:
+        path: "{{ slurmdbd_backup_dir }}"
+        state: directory
+        owner: root
+        group: root
+        mode: "0700"
+
+    - name: Create local fetch directory on Ansible controller
+      ansible.builtin.file:
+        path: "{{ local_fetch_dir }}"
+        state: directory
+        owner: root
+        group: root
+        mode: "0700"
+      delegate_to: localhost
+      become: false
+
+    - name: Validate MariaDB is running
+      ansible.builtin.command:
+        cmd: systemctl is-active mariadb
+      changed_when: false
+
+    - name: Validate SlurmDBD is running
+      ansible.builtin.command:
+        cmd: systemctl is-active slurmdbd
+      changed_when: false
+
+    - name: Validate Slurm accounting database exists
+      ansible.builtin.shell: |
+        set -euo pipefail
+        mysql -N -B -e "SHOW DATABASES LIKE '{{ slurmdbd_storage_loc }}';" | grep -qx "{{ slurmdbd_storage_loc }}"
+      args:
+        executable: /bin/bash
+      changed_when: false
+
+    - name: Dump Slurm accounting database
+      ansible.builtin.shell: |
+        set -euo pipefail
+
+        ts="$(date +%F-%H%M%S)"
+        out="{{ slurmdbd_backup_dir }}/{{ slurmdbd_storage_loc }}-${ts}.sql.gz"
+
+        mysqldump \
+          --single-transaction \
+          --routines \
+          --events \
+          --triggers \
+          {{ slurmdbd_storage_loc }} | gzip -9 > "$out"
+
+        chmod 0600 "$out"
+        echo "$out"
+      args:
+        executable: /bin/bash
+      register: db_dump
+      changed_when: true
+
+    - name: Validate backup file is non-empty
+      ansible.builtin.stat:
+        path: "{{ db_dump.stdout }}"
+      register: backup_file
+
+    - name: Fail if backup file is empty
+      ansible.builtin.fail:
+        msg: "Backup file is empty: {{ db_dump.stdout }}"
+      when: backup_file.stat.size | int < 1024
+
+    - name: Fetch DB backup to Ansible controller
+      ansible.builtin.fetch:
+        src: "{{ db_dump.stdout }}"
+        dest: "{{ local_fetch_dir }}/"
+        flat: true
+
+    - name: Show DB backup result
+      ansible.builtin.debug:
+        msg:
+          - "Remote backup: {{ db_dump.stdout }}"
+          - "Backup size bytes: {{ backup_file.stat.size }}"
+          - "Fetched to: {{ local_fetch_dir }}/"