feat: refactor RAID health check playbook for improved clarity and functionality
This commit is contained in:
martin.fencl
+326
-143
@@ -1,183 +1,366 @@
|
|||||||
# check_raid.yml
|
# check_raid.yml
|
||||||
|
|
||||||
- name: Check Linux MD RAID health on VM via Proxmox
|
- name: Check Linux MD RAID health
|
||||||
hosts: linux_servers
|
hosts: pve2_vm
|
||||||
gather_facts: false
|
gather_facts: false
|
||||||
become: true
|
|
||||||
become_user: root
|
|
||||||
become_method: sudo
|
|
||||||
|
|
||||||
vars:
|
vars:
|
||||||
# VM connection (provided by Semaphore env vars)
|
# RAID_MD can be md0, md1, /dev/md0, or auto
|
||||||
vm_ip: "{{ lookup('env', 'VM_IP') }}"
|
raid_md_raw: "{{ lookup('env', 'RAID_MD') | default('auto', true) | trim }}"
|
||||||
vm_user: "{{ lookup('env', 'VM_USER') }}"
|
raid_md_device: "{{ raid_md_raw | regex_replace('^/dev/', '') }}"
|
||||||
vm_pass: "{{ lookup('env', 'VM_PASS') }}"
|
|
||||||
use_sudo: false
|
|
||||||
|
|
||||||
# Debug mode
|
# Allow check, repair, resync, recovery, or reshape on an otherwise healthy array
|
||||||
DEBUG: "{{ lookup('env', 'DEBUG') | default(0) | int }}"
|
raid_allow_sync: "{{ lookup('env', 'RAID_ALLOW_SYNC') | default('1', true) | int }}"
|
||||||
RETRIES: "{{ lookup('env', 'RETRIES') | default(25) | int }}"
|
|
||||||
|
|
||||||
# RAID specifics
|
# Set to 1 only when a server without MD RAID should be considered healthy
|
||||||
# RAID_MD can be: md0 / md1 / ... OR "auto" to check all arrays found in /proc/mdstat
|
raid_allow_no_array: "{{ lookup('env', 'RAID_ALLOW_NO_ARRAY') | default('0', true) | int }}"
|
||||||
raid_md_device: "{{ lookup('env', 'RAID_MD') | default('md0', true) }}"
|
|
||||||
raid_allow_sync: "{{ lookup('env', 'RAID_ALLOW_SYNC') | default(1, true) | int }}"
|
|
||||||
raid_allow_no_array: "{{ lookup('env', 'RAID_ALLOW_NO_ARRAY') | default(0, true) | int }}"
|
|
||||||
|
|
||||||
raid_retries: "{{ RETRIES }}"
|
# Print raw command data and /proc/mdstat when enabled
|
||||||
raid_delay: 2
|
raid_debug: "{{ lookup('env', 'DEBUG') | default('0', true) | int }}"
|
||||||
ssh_hard_timeout: 30
|
|
||||||
|
|
||||||
# SSH options
|
raid_check_script: |
|
||||||
ssh_opts:
|
import re
|
||||||
- "-o" # English comments
|
import sys
|
||||||
- "StrictHostKeyChecking=no"
|
|
||||||
- "-o"
|
|
||||||
- "UserKnownHostsFile=/dev/null"
|
|
||||||
- "-o"
|
|
||||||
- "GlobalKnownHostsFile=/dev/null"
|
|
||||||
- "-o"
|
|
||||||
- "LogLevel=ERROR"
|
|
||||||
- "-o"
|
|
||||||
- "ConnectTimeout=15"
|
|
||||||
- "-o"
|
|
||||||
- "PreferredAuthentications=password"
|
|
||||||
- "-o"
|
|
||||||
- "PubkeyAuthentication=no"
|
|
||||||
- "-o"
|
|
||||||
- "KbdInteractiveAuthentication=no"
|
|
||||||
- "-o"
|
|
||||||
- "NumberOfPasswordPrompts=1"
|
|
||||||
|
|
||||||
raid_check_cmd: |
|
target = {{ raid_md_device | to_json }}
|
||||||
python3 - <<'PY'
|
allow_sync = bool({{ raid_allow_sync | int }})
|
||||||
# Print exactly one status line and exit with code:
|
allow_no_array = bool({{ raid_allow_no_array | int }})
|
||||||
# 0=OK, 1=FAIL (degraded/disallowed sync), 2=ERROR (unexpected/misconfig)
|
|
||||||
import re, sys
|
|
||||||
|
|
||||||
target = "{{ raid_md_device }}"
|
|
||||||
allow_sync = int("{{ raid_allow_sync }}")
|
|
||||||
allow_no_array = int("{{ raid_allow_no_array }}")
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
txt = open("/proc/mdstat", "r", encoding="utf-8", errors="ignore").read()
|
with open(
|
||||||
except Exception as e:
|
"/proc/mdstat",
|
||||||
print(f"ERROR RAID read_mdstat err={e}")
|
"r",
|
||||||
|
encoding="utf-8",
|
||||||
|
errors="replace",
|
||||||
|
) as mdstat_file:
|
||||||
|
mdstat = mdstat_file.read()
|
||||||
|
except Exception as exc:
|
||||||
|
print(
|
||||||
|
"ERROR RAID "
|
||||||
|
f"read_mdstat_failed error={type(exc).__name__}:{exc}"
|
||||||
|
)
|
||||||
sys.exit(2)
|
sys.exit(2)
|
||||||
|
|
||||||
arrays = {}
|
header_pattern = re.compile(
|
||||||
header_re = re.compile(r"^(md\d+)\s*:\s*active.*$", re.MULTILINE)
|
r"^(md\d+)\s*:\s*(.*)$",
|
||||||
token_re = re.compile(r"^\s*\d+\s+blocks.*\[\d+/\d+\]\s*\[([U_]+)\]\s*$", re.MULTILINE)
|
re.MULTILINE,
|
||||||
|
)
|
||||||
|
header_matches = list(header_pattern.finditer(mdstat))
|
||||||
|
array_blocks = {}
|
||||||
|
|
||||||
for m in header_re.finditer(txt):
|
for index, match in enumerate(header_matches):
|
||||||
name = m.group(1)
|
block_end = (
|
||||||
chunk = txt[m.end():m.end() + 3000]
|
header_matches[index + 1].start()
|
||||||
tm = token_re.search(chunk)
|
if index + 1 < len(header_matches)
|
||||||
if tm:
|
else len(mdstat)
|
||||||
arrays[name] = tm.group(1)
|
)
|
||||||
|
|
||||||
if not arrays:
|
array_blocks[match.group(1)] = mdstat[
|
||||||
|
match.start():block_end
|
||||||
|
]
|
||||||
|
|
||||||
|
def md_sort_key(name):
|
||||||
|
return int(name[2:])
|
||||||
|
|
||||||
|
def parse_array(name, block):
|
||||||
|
header = block.splitlines()[0]
|
||||||
|
header_data = header.split(":", 1)[1].strip()
|
||||||
|
|
||||||
|
state_match = re.match(
|
||||||
|
r"^(active|inactive)\b",
|
||||||
|
header_data,
|
||||||
|
)
|
||||||
|
|
||||||
|
state = (
|
||||||
|
state_match.group(1)
|
||||||
|
if state_match
|
||||||
|
else "unknown"
|
||||||
|
)
|
||||||
|
|
||||||
|
remaining = (
|
||||||
|
header_data[state_match.end():].strip()
|
||||||
|
if state_match
|
||||||
|
else header_data
|
||||||
|
)
|
||||||
|
|
||||||
|
mode = "rw"
|
||||||
|
|
||||||
|
if remaining.startswith("("):
|
||||||
|
closing_parenthesis = remaining.find(")")
|
||||||
|
|
||||||
|
if closing_parenthesis >= 0:
|
||||||
|
mode = remaining[1:closing_parenthesis]
|
||||||
|
remaining = remaining[
|
||||||
|
closing_parenthesis + 1:
|
||||||
|
].strip()
|
||||||
|
|
||||||
|
level = (
|
||||||
|
remaining.split()[0]
|
||||||
|
if remaining
|
||||||
|
else "unknown"
|
||||||
|
)
|
||||||
|
|
||||||
|
status_match = re.search(
|
||||||
|
r"\[(\d+)\s*/\s*(\d+)\]\s*\[([U_]+)\]",
|
||||||
|
block,
|
||||||
|
)
|
||||||
|
|
||||||
|
active_members = None
|
||||||
|
total_members = None
|
||||||
|
member_status = None
|
||||||
|
|
||||||
|
if status_match:
|
||||||
|
active_members = int(status_match.group(1))
|
||||||
|
total_members = int(status_match.group(2))
|
||||||
|
member_status = status_match.group(3)
|
||||||
|
|
||||||
|
operation_match = re.search(
|
||||||
|
r"\b(resync|recovery|reshape|check|repair)\b",
|
||||||
|
block,
|
||||||
|
re.IGNORECASE,
|
||||||
|
)
|
||||||
|
|
||||||
|
operation = (
|
||||||
|
operation_match.group(1).lower()
|
||||||
|
if operation_match
|
||||||
|
else "none"
|
||||||
|
)
|
||||||
|
|
||||||
|
progress_match = re.search(
|
||||||
|
r"(?:resync|recovery|reshape|check|repair)"
|
||||||
|
r"\s*=\s*([0-9.]+%)",
|
||||||
|
block,
|
||||||
|
re.IGNORECASE,
|
||||||
|
)
|
||||||
|
|
||||||
|
progress = (
|
||||||
|
progress_match.group(1)
|
||||||
|
if progress_match
|
||||||
|
else "none"
|
||||||
|
)
|
||||||
|
|
||||||
|
return {
|
||||||
|
"name": name,
|
||||||
|
"state": state,
|
||||||
|
"mode": mode,
|
||||||
|
"level": level,
|
||||||
|
"active_members": active_members,
|
||||||
|
"total_members": total_members,
|
||||||
|
"member_status": member_status,
|
||||||
|
"operation": operation,
|
||||||
|
"progress": progress,
|
||||||
|
"faulty_member": bool(
|
||||||
|
re.search(r"\(F\)", block)
|
||||||
|
),
|
||||||
|
}
|
||||||
|
|
||||||
|
def format_array(array):
|
||||||
|
if array["member_status"] is None:
|
||||||
|
members = "n/a"
|
||||||
|
status = "n/a"
|
||||||
|
else:
|
||||||
|
members = (
|
||||||
|
f'{array["active_members"]}/'
|
||||||
|
f'{array["total_members"]}'
|
||||||
|
)
|
||||||
|
status = array["member_status"]
|
||||||
|
|
||||||
|
return (
|
||||||
|
f'{array["name"]}'
|
||||||
|
"{"
|
||||||
|
f'state={array["state"]},'
|
||||||
|
f'mode={array["mode"]},'
|
||||||
|
f'level={array["level"]},'
|
||||||
|
f'members={members},'
|
||||||
|
f'status={status},'
|
||||||
|
f'operation={array["operation"]},'
|
||||||
|
f'progress={array["progress"]}'
|
||||||
|
"}"
|
||||||
|
)
|
||||||
|
|
||||||
|
if not array_blocks:
|
||||||
if allow_no_array:
|
if allow_no_array:
|
||||||
print("OK RAID none=no-md-arrays")
|
print("OK RAID none=no-md-arrays")
|
||||||
sys.exit(0)
|
sys.exit(0)
|
||||||
|
|
||||||
print("ERROR RAID none=no-md-arrays")
|
print("ERROR RAID none=no-md-arrays")
|
||||||
sys.exit(2)
|
sys.exit(2)
|
||||||
|
|
||||||
syncing = bool(re.search(r"\b(resync|recovery|reshape|check|repair)\b", txt))
|
available_arrays = sorted(
|
||||||
|
array_blocks,
|
||||||
|
key=md_sort_key,
|
||||||
|
)
|
||||||
|
|
||||||
if target == "auto":
|
if target == "auto":
|
||||||
to_check = sorted(arrays.keys())
|
selected_names = available_arrays
|
||||||
|
elif target in array_blocks:
|
||||||
|
selected_names = [target]
|
||||||
else:
|
else:
|
||||||
if target not in arrays:
|
print(
|
||||||
found = ",".join(sorted(arrays.keys()))
|
"ERROR RAID "
|
||||||
print(f"ERROR RAID target_not_found target={target} found={found}")
|
f"target_not_found target={target} "
|
||||||
sys.exit(2)
|
f'found={",".join(available_arrays)}'
|
||||||
to_check = [target]
|
)
|
||||||
|
sys.exit(2)
|
||||||
|
|
||||||
tokens_str = " ".join([f"{name}=[{arrays[name]}]" for name in to_check])
|
selected_arrays = [
|
||||||
degraded = any("_" in arrays[name] for name in to_check)
|
parse_array(name, array_blocks[name])
|
||||||
|
for name in selected_names
|
||||||
|
]
|
||||||
|
|
||||||
if degraded:
|
failures = []
|
||||||
print(f"FAIL RAID {tokens_str} syncing={int(syncing)}")
|
errors = []
|
||||||
|
|
||||||
|
# RAID0 and linear arrays do not normally contain [UU] status tokens
|
||||||
|
tokenless_levels = {
|
||||||
|
"raid0",
|
||||||
|
"linear",
|
||||||
|
}
|
||||||
|
|
||||||
|
for array in selected_arrays:
|
||||||
|
name = array["name"]
|
||||||
|
|
||||||
|
if array["state"] != "active":
|
||||||
|
failures.append(
|
||||||
|
f'{name}:state={array["state"]}'
|
||||||
|
)
|
||||||
|
|
||||||
|
if array["faulty_member"]:
|
||||||
|
failures.append(
|
||||||
|
f"{name}:faulty-member"
|
||||||
|
)
|
||||||
|
|
||||||
|
if array["member_status"] is not None:
|
||||||
|
if "_" in array["member_status"]:
|
||||||
|
failures.append(
|
||||||
|
f'{name}:degraded='
|
||||||
|
f'[{array["member_status"]}]'
|
||||||
|
)
|
||||||
|
|
||||||
|
if (
|
||||||
|
array["active_members"]
|
||||||
|
!= array["total_members"]
|
||||||
|
):
|
||||||
|
failures.append(
|
||||||
|
f'{name}:members='
|
||||||
|
f'{array["active_members"]}/'
|
||||||
|
f'{array["total_members"]}'
|
||||||
|
)
|
||||||
|
|
||||||
|
elif array["level"] not in tokenless_levels:
|
||||||
|
errors.append(
|
||||||
|
f"{name}:member-status-not-found"
|
||||||
|
)
|
||||||
|
|
||||||
|
if (
|
||||||
|
array["operation"] != "none"
|
||||||
|
and not allow_sync
|
||||||
|
):
|
||||||
|
failures.append(
|
||||||
|
f'{name}:operation={array["operation"]}'
|
||||||
|
)
|
||||||
|
|
||||||
|
details = " ".join(
|
||||||
|
format_array(array)
|
||||||
|
for array in selected_arrays
|
||||||
|
)
|
||||||
|
|
||||||
|
if errors:
|
||||||
|
print(
|
||||||
|
f"ERROR RAID {details} "
|
||||||
|
f'reason={";".join(errors)}'
|
||||||
|
)
|
||||||
|
sys.exit(2)
|
||||||
|
|
||||||
|
if failures:
|
||||||
|
print(
|
||||||
|
f"FAIL RAID {details} "
|
||||||
|
f'reason={";".join(failures)}'
|
||||||
|
)
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
if syncing and not allow_sync:
|
print(f"OK RAID {details}")
|
||||||
print(f"FAIL RAID {tokens_str} syncing={int(syncing)} allow_sync={allow_sync}")
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
print(f"OK RAID {tokens_str} syncing={int(syncing)}")
|
|
||||||
sys.exit(0)
|
sys.exit(0)
|
||||||
PY
|
|
||||||
|
|
||||||
tasks:
|
tasks:
|
||||||
- name: Ensure sshpass is installed (for password-based SSH) # English comments
|
- name: Validate RAID check configuration
|
||||||
ansible.builtin.apt:
|
|
||||||
name: sshpass
|
|
||||||
state: present
|
|
||||||
update_cache: yes
|
|
||||||
run_once: true
|
|
||||||
|
|
||||||
- name: Run RAID check on VM (via SSH) # single command, no loop
|
|
||||||
ansible.builtin.command:
|
|
||||||
argv: >-
|
|
||||||
{{
|
|
||||||
['timeout', '-k', '5', (ssh_hard_timeout | string)]
|
|
||||||
+ ['sshpass', '-e', 'ssh']
|
|
||||||
+ ssh_opts
|
|
||||||
+ [ vm_user ~ '@' ~ vm_ip,
|
|
||||||
'bash', '-lc',
|
|
||||||
('sudo ' if use_sudo else '') + raid_check_cmd
|
|
||||||
]
|
|
||||||
}}
|
|
||||||
environment:
|
|
||||||
SSHPASS: "{{ vm_pass }}"
|
|
||||||
register: raid_cmd
|
|
||||||
changed_when: false
|
|
||||||
failed_when: false # we decide via assert below
|
|
||||||
retries: "{{ raid_retries }}"
|
|
||||||
delay: "{{ raid_delay }}"
|
|
||||||
until: raid_cmd.rc not in [124, 255]
|
|
||||||
run_once: true
|
|
||||||
|
|
||||||
- name: Build one-line summary (always)
|
|
||||||
ansible.builtin.set_fact:
|
|
||||||
raid_line: >-
|
|
||||||
{{
|
|
||||||
(raid_cmd.stdout | default('') | trim)
|
|
||||||
if ((raid_cmd.stdout | default('') | trim) | length) > 0
|
|
||||||
else ('ERROR RAID no-output rc=' ~ (raid_cmd.rc | string))
|
|
||||||
}}
|
|
||||||
changed_when: false
|
|
||||||
run_once: true
|
|
||||||
|
|
||||||
- name: RAID result (always one line)
|
|
||||||
ansible.builtin.assert:
|
ansible.builtin.assert:
|
||||||
that:
|
that:
|
||||||
- raid_cmd.rc == 0
|
- (raid_allow_sync | int) in [0, 1]
|
||||||
success_msg: "{{ raid_line }}"
|
- (raid_allow_no_array | int) in [0, 1]
|
||||||
fail_msg: "{{ raid_line }}"
|
- (raid_debug | int) in [0, 1]
|
||||||
run_once: true
|
- >-
|
||||||
|
(raid_md_device |
|
||||||
|
regex_search('^(auto|md[0-9]+)$'))
|
||||||
|
is not none
|
||||||
|
fail_msg: >-
|
||||||
|
ERROR RAID invalid-configuration.
|
||||||
|
RAID_MD must be auto, md0, md1, or /dev/md0.
|
||||||
|
quiet: true
|
||||||
|
|
||||||
# Optional verbose debug
|
- name: Run Linux MD RAID health check
|
||||||
- name: Debug | /proc/mdstat (VM)
|
|
||||||
ansible.builtin.command:
|
ansible.builtin.command:
|
||||||
argv: >-
|
argv:
|
||||||
{{
|
- python3
|
||||||
['timeout', '-k', '5', (ssh_hard_timeout | string)]
|
- "-"
|
||||||
+ ['sshpass', '-e', 'ssh']
|
stdin: "{{ raid_check_script }}"
|
||||||
+ ssh_opts
|
stdin_add_newline: true
|
||||||
+ [ vm_user ~ '@' ~ vm_ip, 'bash', '-lc', "cat /proc/mdstat" ]
|
register: raid_cmd
|
||||||
}}
|
|
||||||
environment:
|
|
||||||
SSHPASS: "{{ vm_pass }}"
|
|
||||||
register: mdstat_dbg
|
|
||||||
changed_when: false
|
changed_when: false
|
||||||
failed_when: false
|
failed_when: false
|
||||||
when: DEBUG == 1
|
|
||||||
run_once: true
|
|
||||||
|
|
||||||
- name: Debug | mdstat output
|
- name: Build one-line RAID result
|
||||||
|
ansible.builtin.set_fact:
|
||||||
|
raid_line: >-
|
||||||
|
{%- set stdout = (
|
||||||
|
raid_cmd.stdout
|
||||||
|
| default('')
|
||||||
|
| trim
|
||||||
|
| regex_replace('[\r\n]+', ' ')
|
||||||
|
| regex_replace('\s+', ' ')
|
||||||
|
) -%}
|
||||||
|
{%- set stderr = (
|
||||||
|
raid_cmd.stderr
|
||||||
|
| default('')
|
||||||
|
| trim
|
||||||
|
| regex_replace('[\r\n]+', ' ')
|
||||||
|
| regex_replace('\s+', ' ')
|
||||||
|
) -%}
|
||||||
|
{%- if stdout | length > 0 -%}
|
||||||
|
{{ stdout }}
|
||||||
|
{%- elif stderr | length > 0 -%}
|
||||||
|
ERROR RAID command_failed rc={{ raid_cmd.rc }} stderr={{ stderr[:500] }}
|
||||||
|
{%- else -%}
|
||||||
|
ERROR RAID no-output rc={{ raid_cmd.rc }}
|
||||||
|
{%- endif -%}
|
||||||
|
changed_when: false
|
||||||
|
|
||||||
|
- name: Show raw RAID command result
|
||||||
ansible.builtin.debug:
|
ansible.builtin.debug:
|
||||||
msg: "{{ mdstat_dbg.stdout | default('') }}"
|
msg:
|
||||||
when: DEBUG == 1
|
rc: "{{ raid_cmd.rc }}"
|
||||||
run_once: true
|
stdout: "{{ raid_cmd.stdout | default('') }}"
|
||||||
|
stderr: "{{ raid_cmd.stderr | default('') }}"
|
||||||
|
when: (raid_debug | int) == 1
|
||||||
|
|
||||||
|
- name: Read raw mdstat in debug mode
|
||||||
|
ansible.builtin.command:
|
||||||
|
argv:
|
||||||
|
- cat
|
||||||
|
- /proc/mdstat
|
||||||
|
register: mdstat_debug
|
||||||
|
changed_when: false
|
||||||
|
when: (raid_debug | int) == 1
|
||||||
|
|
||||||
|
- name: Show raw mdstat in debug mode
|
||||||
|
ansible.builtin.debug:
|
||||||
|
msg: "{{ mdstat_debug.stdout | default('') }}"
|
||||||
|
when: (raid_debug | int) == 1
|
||||||
|
|
||||||
|
- name: RAID result
|
||||||
|
ansible.builtin.assert:
|
||||||
|
that:
|
||||||
|
- (raid_cmd.rc | int) == 0
|
||||||
|
success_msg: "{{ raid_line }}"
|
||||||
|
fail_msg: "{{ raid_line }}"
|
||||||
|
quiet: true
|
||||||
Reference in New Issue
Block a user