From bb45e70af346c4143dbda82a155fb4e099f9e341 Mon Sep 17 00:00:00 2001 From: "martin.fencl" Date: Wed, 29 Jul 2026 14:56:11 +0200 Subject: [PATCH] feat: refactor RAID health check playbook for improved clarity and functionality --- check_raid.yml | 469 ++++++++++++++++++++++++++++++++++--------------- 1 file changed, 326 insertions(+), 143 deletions(-) diff --git a/check_raid.yml b/check_raid.yml index 12a00ff..b02fe4a 100644 --- a/check_raid.yml +++ b/check_raid.yml @@ -1,183 +1,366 @@ # check_raid.yml -- name: Check Linux MD RAID health on VM via Proxmox - hosts: linux_servers +- name: Check Linux MD RAID health + hosts: pve2_vm gather_facts: false - become: true - become_user: root - become_method: sudo vars: - # VM connection (provided by Semaphore env vars) - vm_ip: "{{ lookup('env', 'VM_IP') }}" - vm_user: "{{ lookup('env', 'VM_USER') }}" - vm_pass: "{{ lookup('env', 'VM_PASS') }}" - use_sudo: false + # RAID_MD can be md0, md1, /dev/md0, or auto + raid_md_raw: "{{ lookup('env', 'RAID_MD') | default('auto', true) | trim }}" + raid_md_device: "{{ raid_md_raw | regex_replace('^/dev/', '') }}" - # Debug mode - DEBUG: "{{ lookup('env', 'DEBUG') | default(0) | int }}" - RETRIES: "{{ lookup('env', 'RETRIES') | default(25) | int }}" + # Allow check, repair, resync, recovery, or reshape on an otherwise healthy array + raid_allow_sync: "{{ lookup('env', 'RAID_ALLOW_SYNC') | default('1', true) | int }}" - # RAID specifics - # RAID_MD can be: md0 / md1 / ... OR "auto" to check all arrays found in /proc/mdstat - raid_md_device: "{{ lookup('env', 'RAID_MD') | default('md0', true) }}" - raid_allow_sync: "{{ lookup('env', 'RAID_ALLOW_SYNC') | default(1, true) | int }}" - raid_allow_no_array: "{{ lookup('env', 'RAID_ALLOW_NO_ARRAY') | default(0, true) | int }}" + # Set to 1 only when a server without MD RAID should be considered healthy + raid_allow_no_array: "{{ lookup('env', 'RAID_ALLOW_NO_ARRAY') | default('0', true) | int }}" - raid_retries: "{{ RETRIES }}" - raid_delay: 2 - ssh_hard_timeout: 30 + # Print raw command data and /proc/mdstat when enabled + raid_debug: "{{ lookup('env', 'DEBUG') | default('0', true) | int }}" - # SSH options - ssh_opts: - - "-o" # English comments - - "StrictHostKeyChecking=no" - - "-o" - - "UserKnownHostsFile=/dev/null" - - "-o" - - "GlobalKnownHostsFile=/dev/null" - - "-o" - - "LogLevel=ERROR" - - "-o" - - "ConnectTimeout=15" - - "-o" - - "PreferredAuthentications=password" - - "-o" - - "PubkeyAuthentication=no" - - "-o" - - "KbdInteractiveAuthentication=no" - - "-o" - - "NumberOfPasswordPrompts=1" + raid_check_script: | + import re + import sys - raid_check_cmd: | - python3 - <<'PY' - # Print exactly one status line and exit with code: - # 0=OK, 1=FAIL (degraded/disallowed sync), 2=ERROR (unexpected/misconfig) - import re, sys - - target = "{{ raid_md_device }}" - allow_sync = int("{{ raid_allow_sync }}") - allow_no_array = int("{{ raid_allow_no_array }}") + target = {{ raid_md_device | to_json }} + allow_sync = bool({{ raid_allow_sync | int }}) + allow_no_array = bool({{ raid_allow_no_array | int }}) try: - txt = open("/proc/mdstat", "r", encoding="utf-8", errors="ignore").read() - except Exception as e: - print(f"ERROR RAID read_mdstat err={e}") + with open( + "/proc/mdstat", + "r", + encoding="utf-8", + errors="replace", + ) as mdstat_file: + mdstat = mdstat_file.read() + except Exception as exc: + print( + "ERROR RAID " + f"read_mdstat_failed error={type(exc).__name__}:{exc}" + ) sys.exit(2) - arrays = {} - header_re = re.compile(r"^(md\d+)\s*:\s*active.*$", re.MULTILINE) - token_re = re.compile(r"^\s*\d+\s+blocks.*\[\d+/\d+\]\s*\[([U_]+)\]\s*$", re.MULTILINE) + header_pattern = re.compile( + r"^(md\d+)\s*:\s*(.*)$", + re.MULTILINE, + ) + header_matches = list(header_pattern.finditer(mdstat)) + array_blocks = {} - for m in header_re.finditer(txt): - name = m.group(1) - chunk = txt[m.end():m.end() + 3000] - tm = token_re.search(chunk) - if tm: - arrays[name] = tm.group(1) + for index, match in enumerate(header_matches): + block_end = ( + header_matches[index + 1].start() + if index + 1 < len(header_matches) + else len(mdstat) + ) - if not arrays: + array_blocks[match.group(1)] = mdstat[ + match.start():block_end + ] + + def md_sort_key(name): + return int(name[2:]) + + def parse_array(name, block): + header = block.splitlines()[0] + header_data = header.split(":", 1)[1].strip() + + state_match = re.match( + r"^(active|inactive)\b", + header_data, + ) + + state = ( + state_match.group(1) + if state_match + else "unknown" + ) + + remaining = ( + header_data[state_match.end():].strip() + if state_match + else header_data + ) + + mode = "rw" + + if remaining.startswith("("): + closing_parenthesis = remaining.find(")") + + if closing_parenthesis >= 0: + mode = remaining[1:closing_parenthesis] + remaining = remaining[ + closing_parenthesis + 1: + ].strip() + + level = ( + remaining.split()[0] + if remaining + else "unknown" + ) + + status_match = re.search( + r"\[(\d+)\s*/\s*(\d+)\]\s*\[([U_]+)\]", + block, + ) + + active_members = None + total_members = None + member_status = None + + if status_match: + active_members = int(status_match.group(1)) + total_members = int(status_match.group(2)) + member_status = status_match.group(3) + + operation_match = re.search( + r"\b(resync|recovery|reshape|check|repair)\b", + block, + re.IGNORECASE, + ) + + operation = ( + operation_match.group(1).lower() + if operation_match + else "none" + ) + + progress_match = re.search( + r"(?:resync|recovery|reshape|check|repair)" + r"\s*=\s*([0-9.]+%)", + block, + re.IGNORECASE, + ) + + progress = ( + progress_match.group(1) + if progress_match + else "none" + ) + + return { + "name": name, + "state": state, + "mode": mode, + "level": level, + "active_members": active_members, + "total_members": total_members, + "member_status": member_status, + "operation": operation, + "progress": progress, + "faulty_member": bool( + re.search(r"\(F\)", block) + ), + } + + def format_array(array): + if array["member_status"] is None: + members = "n/a" + status = "n/a" + else: + members = ( + f'{array["active_members"]}/' + f'{array["total_members"]}' + ) + status = array["member_status"] + + return ( + f'{array["name"]}' + "{" + f'state={array["state"]},' + f'mode={array["mode"]},' + f'level={array["level"]},' + f'members={members},' + f'status={status},' + f'operation={array["operation"]},' + f'progress={array["progress"]}' + "}" + ) + + if not array_blocks: if allow_no_array: print("OK RAID none=no-md-arrays") sys.exit(0) + print("ERROR RAID none=no-md-arrays") sys.exit(2) - syncing = bool(re.search(r"\b(resync|recovery|reshape|check|repair)\b", txt)) + available_arrays = sorted( + array_blocks, + key=md_sort_key, + ) if target == "auto": - to_check = sorted(arrays.keys()) + selected_names = available_arrays + elif target in array_blocks: + selected_names = [target] else: - if target not in arrays: - found = ",".join(sorted(arrays.keys())) - print(f"ERROR RAID target_not_found target={target} found={found}") - sys.exit(2) - to_check = [target] + print( + "ERROR RAID " + f"target_not_found target={target} " + f'found={",".join(available_arrays)}' + ) + sys.exit(2) - tokens_str = " ".join([f"{name}=[{arrays[name]}]" for name in to_check]) - degraded = any("_" in arrays[name] for name in to_check) + selected_arrays = [ + parse_array(name, array_blocks[name]) + for name in selected_names + ] - if degraded: - print(f"FAIL RAID {tokens_str} syncing={int(syncing)}") + failures = [] + errors = [] + + # RAID0 and linear arrays do not normally contain [UU] status tokens + tokenless_levels = { + "raid0", + "linear", + } + + for array in selected_arrays: + name = array["name"] + + if array["state"] != "active": + failures.append( + f'{name}:state={array["state"]}' + ) + + if array["faulty_member"]: + failures.append( + f"{name}:faulty-member" + ) + + if array["member_status"] is not None: + if "_" in array["member_status"]: + failures.append( + f'{name}:degraded=' + f'[{array["member_status"]}]' + ) + + if ( + array["active_members"] + != array["total_members"] + ): + failures.append( + f'{name}:members=' + f'{array["active_members"]}/' + f'{array["total_members"]}' + ) + + elif array["level"] not in tokenless_levels: + errors.append( + f"{name}:member-status-not-found" + ) + + if ( + array["operation"] != "none" + and not allow_sync + ): + failures.append( + f'{name}:operation={array["operation"]}' + ) + + details = " ".join( + format_array(array) + for array in selected_arrays + ) + + if errors: + print( + f"ERROR RAID {details} " + f'reason={";".join(errors)}' + ) + sys.exit(2) + + if failures: + print( + f"FAIL RAID {details} " + f'reason={";".join(failures)}' + ) sys.exit(1) - if syncing and not allow_sync: - print(f"FAIL RAID {tokens_str} syncing={int(syncing)} allow_sync={allow_sync}") - sys.exit(1) - - print(f"OK RAID {tokens_str} syncing={int(syncing)}") + print(f"OK RAID {details}") sys.exit(0) - PY tasks: - - name: Ensure sshpass is installed (for password-based SSH) # English comments - ansible.builtin.apt: - name: sshpass - state: present - update_cache: yes - run_once: true - - - name: Run RAID check on VM (via SSH) # single command, no loop - ansible.builtin.command: - argv: >- - {{ - ['timeout', '-k', '5', (ssh_hard_timeout | string)] - + ['sshpass', '-e', 'ssh'] - + ssh_opts - + [ vm_user ~ '@' ~ vm_ip, - 'bash', '-lc', - ('sudo ' if use_sudo else '') + raid_check_cmd - ] - }} - environment: - SSHPASS: "{{ vm_pass }}" - register: raid_cmd - changed_when: false - failed_when: false # we decide via assert below - retries: "{{ raid_retries }}" - delay: "{{ raid_delay }}" - until: raid_cmd.rc not in [124, 255] - run_once: true - - - name: Build one-line summary (always) - ansible.builtin.set_fact: - raid_line: >- - {{ - (raid_cmd.stdout | default('') | trim) - if ((raid_cmd.stdout | default('') | trim) | length) > 0 - else ('ERROR RAID no-output rc=' ~ (raid_cmd.rc | string)) - }} - changed_when: false - run_once: true - - - name: RAID result (always one line) + - name: Validate RAID check configuration ansible.builtin.assert: that: - - raid_cmd.rc == 0 - success_msg: "{{ raid_line }}" - fail_msg: "{{ raid_line }}" - run_once: true + - (raid_allow_sync | int) in [0, 1] + - (raid_allow_no_array | int) in [0, 1] + - (raid_debug | int) in [0, 1] + - >- + (raid_md_device | + regex_search('^(auto|md[0-9]+)$')) + is not none + fail_msg: >- + ERROR RAID invalid-configuration. + RAID_MD must be auto, md0, md1, or /dev/md0. + quiet: true - # Optional verbose debug - - name: Debug | /proc/mdstat (VM) + - name: Run Linux MD RAID health check ansible.builtin.command: - argv: >- - {{ - ['timeout', '-k', '5', (ssh_hard_timeout | string)] - + ['sshpass', '-e', 'ssh'] - + ssh_opts - + [ vm_user ~ '@' ~ vm_ip, 'bash', '-lc', "cat /proc/mdstat" ] - }} - environment: - SSHPASS: "{{ vm_pass }}" - register: mdstat_dbg + argv: + - python3 + - "-" + stdin: "{{ raid_check_script }}" + stdin_add_newline: true + register: raid_cmd changed_when: false failed_when: false - when: DEBUG == 1 - run_once: true - - name: Debug | mdstat output + - name: Build one-line RAID result + ansible.builtin.set_fact: + raid_line: >- + {%- set stdout = ( + raid_cmd.stdout + | default('') + | trim + | regex_replace('[\r\n]+', ' ') + | regex_replace('\s+', ' ') + ) -%} + {%- set stderr = ( + raid_cmd.stderr + | default('') + | trim + | regex_replace('[\r\n]+', ' ') + | regex_replace('\s+', ' ') + ) -%} + {%- if stdout | length > 0 -%} + {{ stdout }} + {%- elif stderr | length > 0 -%} + ERROR RAID command_failed rc={{ raid_cmd.rc }} stderr={{ stderr[:500] }} + {%- else -%} + ERROR RAID no-output rc={{ raid_cmd.rc }} + {%- endif -%} + changed_when: false + + - name: Show raw RAID command result ansible.builtin.debug: - msg: "{{ mdstat_dbg.stdout | default('') }}" - when: DEBUG == 1 - run_once: true + msg: + rc: "{{ raid_cmd.rc }}" + stdout: "{{ raid_cmd.stdout | default('') }}" + stderr: "{{ raid_cmd.stderr | default('') }}" + when: (raid_debug | int) == 1 + + - name: Read raw mdstat in debug mode + ansible.builtin.command: + argv: + - cat + - /proc/mdstat + register: mdstat_debug + changed_when: false + when: (raid_debug | int) == 1 + + - name: Show raw mdstat in debug mode + ansible.builtin.debug: + msg: "{{ mdstat_debug.stdout | default('') }}" + when: (raid_debug | int) == 1 + + - name: RAID result + ansible.builtin.assert: + that: + - (raid_cmd.rc | int) == 0 + success_msg: "{{ raid_line }}" + fail_msg: "{{ raid_line }}" + quiet: true \ No newline at end of file