ansible_fencl/check_raid.yml

# check_raid.yml

---
- name: Check Linux MD RAID health on pve1
  hosts: pve1_vm
  gather_facts: false

  vars:
    # Semaphore Survey Variables are preferred.
    # Environment variables are used only as a fallback.
    raid_md_raw: >-
      {{
        RAID_MD
        | default(lookup('env', 'RAID_MD'), true)
        | default('auto', true)
        | string
        | trim
        | lower
      }}

    # RAID_MD can be md0, md1, /dev/md0, or auto
    raid_md_device: "{{ raid_md_raw | regex_replace('^/dev/', '') }}"

    # Allow a healthy array to be checking, repairing, recovering,
    # resyncing, or reshaping
    raid_allow_sync: >-
      {{
        RAID_ALLOW_SYNC
        | default(lookup('env', 'RAID_ALLOW_SYNC'), true)
        | default('1', true)
        | int
      }}

    # A server without Linux MD RAID is considered unhealthy by default
    raid_allow_no_array: >-
      {{
        RAID_ALLOW_NO_ARRAY
        | default(lookup('env', 'RAID_ALLOW_NO_ARRAY'), true)
        | default('0', true)
        | int
      }}

    # Enable raw diagnostic output
    raid_debug: >-
      {{
        DEBUG
        | default(lookup('env', 'DEBUG'), true)
        | default('0', true)
        | int
      }}

    raid_check_script: |
      import re
      import sys

      target = {{ raid_md_device | to_json }}
      allow_sync = bool({{ raid_allow_sync | int }})
      allow_no_array = bool({{ raid_allow_no_array | int }})

      try:
          with open(
              "/proc/mdstat",
              "r",
              encoding="utf-8",
              errors="replace",
          ) as mdstat_file:
              mdstat = mdstat_file.read()
      except Exception as exc:
          print(
              "ERROR RAID "
              f"read_mdstat_failed="
              f"{type(exc).__name__}:{exc}"
          )
          sys.exit(2)

      header_pattern = re.compile(
          r"^(md\d+)\s*:\s*(.*)$",
          re.MULTILINE,
      )

      header_matches = list(header_pattern.finditer(mdstat))
      array_blocks = {}

      for index, match in enumerate(header_matches):
          block_end = (
              header_matches[index + 1].start()
              if index + 1 < len(header_matches)
              else len(mdstat)
          )

          array_blocks[match.group(1)] = mdstat[
              match.start():block_end
          ]

      def md_sort_key(name):
          try:
              return int(name[2:])
          except ValueError:
              return name

      def parse_array(name, block):
          lines = block.splitlines()
          header = lines[0]
          header_data = header.split(":", 1)[1].strip()

          state_match = re.match(
              r"^(active|inactive)\b",
              header_data,
              re.IGNORECASE,
          )

          state = (
              state_match.group(1).lower()
              if state_match
              else "unknown"
          )

          remaining_header = (
              header_data[state_match.end():].strip()
              if state_match
              else header_data
          )

          mode = "rw"

          # Handle states such as "active (auto-read-only)"
          if remaining_header.startswith("("):
              closing_parenthesis = remaining_header.find(")")

              if closing_parenthesis >= 0:
                  mode = remaining_header[
                      1:closing_parenthesis
                  ]

                  remaining_header = remaining_header[
                      closing_parenthesis + 1:
                  ].strip()

          level = (
              remaining_header.split()[0].lower()
              if remaining_header
              else "unknown"
          )

          status_match = re.search(
              r"\[(\d+)\s*/\s*(\d+)\]"
              r"\s*\[([U_]+)\]",
              block,
          )

          active_members = None
          total_members = None
          member_status = None

          if status_match:
              active_members = int(status_match.group(1))
              total_members = int(status_match.group(2))
              member_status = status_match.group(3)

          operation_match = re.search(
              r"\b("
              r"resync|"
              r"recovery|"
              r"reshape|"
              r"check|"
              r"repair"
              r")\b",
              block,
              re.IGNORECASE,
          )

          operation = (
              operation_match.group(1).lower()
              if operation_match
              else "none"
          )

          progress_match = re.search(
              r"(?:"
              r"resync|"
              r"recovery|"
              r"reshape|"
              r"check|"
              r"repair"
              r")\s*=\s*([0-9.]+%)",
              block,
              re.IGNORECASE,
          )

          progress = (
              progress_match.group(1)
              if progress_match
              else "none"
          )

          return {
              "name": name,
              "state": state,
              "mode": mode,
              "level": level,
              "active_members": active_members,
              "total_members": total_members,
              "member_status": member_status,
              "operation": operation,
              "progress": progress,
              "faulty_member": bool(
                  re.search(
                      r"\(F\)",
                      block,
                      re.IGNORECASE,
                  )
              ),
          }

      def format_array(array):
          if array["member_status"] is None:
              members = "n/a"
              member_status = "n/a"
          else:
              members = (
                  f'{array["active_members"]}/'
                  f'{array["total_members"]}'
              )

              member_status = array["member_status"]

          return (
              f'{array["name"]}'
              "{"
              f'state={array["state"]},'
              f'mode={array["mode"]},'
              f'level={array["level"]},'
              f'members={members},'
              f'status={member_status},'
              f'operation={array["operation"]},'
              f'progress={array["progress"]}'
              "}"
          )

      if not array_blocks:
          if allow_no_array:
              print("OK RAID none=no-md-arrays")
              sys.exit(0)

          print("ERROR RAID none=no-md-arrays")
          sys.exit(2)

      available_arrays = sorted(
          array_blocks.keys(),
          key=md_sort_key,
      )

      if target == "auto":
          selected_names = available_arrays
      elif target in array_blocks:
          selected_names = [target]
      else:
          print(
              "ERROR RAID "
              f"target_not_found={target} "
              f'found={",".join(available_arrays)}'
          )
          sys.exit(2)

      selected_arrays = [
          parse_array(
              name,
              array_blocks[name],
          )
          for name in selected_names
      ]

      failures = []
      errors = []

      # These RAID levels may not always provide a redundant-member token
      tokenless_levels = {
          "raid0",
          "linear",
      }

      for array in selected_arrays:
          name = array["name"]

          if array["state"] != "active":
              failures.append(
                  f'{name}:state={array["state"]}'
              )

          if array["faulty_member"]:
              failures.append(
                  f"{name}:faulty-member"
              )

          if array["member_status"] is not None:
              if "_" in array["member_status"]:
                  failures.append(
                      f'{name}:degraded='
                      f'[{array["member_status"]}]'
                  )

              if (
                  array["active_members"]
                  != array["total_members"]
              ):
                  failures.append(
                      f'{name}:members='
                      f'{array["active_members"]}/'
                      f'{array["total_members"]}'
                  )

          elif array["level"] not in tokenless_levels:
              errors.append(
                  f"{name}:member-status-not-found"
              )

          if (
              array["operation"] != "none"
              and not allow_sync
          ):
              failures.append(
                  f'{name}:operation='
                  f'{array["operation"]}'
              )

      details = " ".join(
          format_array(array)
          for array in selected_arrays
      )

      if errors:
          print(
              f"ERROR RAID {details} "
              f'reason={";".join(errors)}'
          )
          sys.exit(2)

      if failures:
          print(
              f"FAIL RAID {details} "
              f'reason={";".join(failures)}'
          )
          sys.exit(1)

      print(f"OK RAID {details}")
      sys.exit(0)

  tasks:
    - name: Validate RAID check configuration
      ansible.builtin.assert:
        that:
          - (raid_allow_sync | int) in [0, 1]
          - (raid_allow_no_array | int) in [0, 1]
          - (raid_debug | int) in [0, 1]
          - >-
            (
              raid_md_device
              | regex_search('^(auto|md[0-9]+)$')
            ) is not none
        fail_msg: >-
          ERROR RAID invalid-configuration.
          RAID_MD must be auto, md0, md1, or /dev/md0.
          DEBUG, RAID_ALLOW_SYNC, and RAID_ALLOW_NO_ARRAY
          must be 0 or 1.
        quiet: true

    - name: Run Linux MD RAID health check
      ansible.builtin.command:
        argv:
          - python3
          - "-"
        stdin: "{{ raid_check_script }}"
        stdin_add_newline: true
      register: raid_cmd
      changed_when: false
      failed_when: false

    - name: Build one-line RAID result
      ansible.builtin.set_fact:
        raid_line: >-
          {%- set stdout = (
            raid_cmd.stdout
            | default('')
            | trim
            | regex_replace('[\r\n]+', ' ')
            | regex_replace('\s+', ' ')
          ) -%}
          {%- set stderr = (
            raid_cmd.stderr
            | default('')
            | trim
            | regex_replace('[\r\n]+', ' ')
            | regex_replace('\s+', ' ')
          ) -%}
          {%- if stdout | length > 0 -%}
          {{ stdout }}
          {%- elif stderr | length > 0 -%}
          ERROR RAID command_failed rc={{ raid_cmd.rc }}
          stderr={{ stderr[:500] }}
          {%- else -%}
          ERROR RAID no-output rc={{ raid_cmd.rc }}
          {%- endif -%}
      changed_when: false

    - name: Show raw RAID command result
      ansible.builtin.debug:
        msg:
          rc: "{{ raid_cmd.rc }}"
          stdout: "{{ raid_cmd.stdout | default('') }}"
          stderr: "{{ raid_cmd.stderr | default('') }}"
      when: (raid_debug | int) == 1

    - name: Read raw mdstat in debug mode
      ansible.builtin.command:
        argv:
          - cat
          - /proc/mdstat
      register: mdstat_debug
      changed_when: false
      failed_when: false
      when: (raid_debug | int) == 1

    - name: Show raw mdstat in debug mode
      ansible.builtin.debug:
        msg:
          rc: "{{ mdstat_debug.rc | default('undefined') }}"
          stdout: "{{ mdstat_debug.stdout | default('') }}"
          stderr: "{{ mdstat_debug.stderr | default('') }}"
      when: (raid_debug | int) == 1

    - name: Show successful RAID result
      ansible.builtin.debug:
        msg: "{{ raid_line }}"
      when: (raid_cmd.rc | int) == 0

    - name: Fail when RAID is unhealthy
      ansible.builtin.fail:
        msg: "{{ raid_line }}"
      when: (raid_cmd.rc | int) != 0