Merge pull request 'Manage own borgmatic systemd service and timer' (#1) from backup-systemd-units into main

Reviewed-on: #1

.gitignore

@@ -0,0 +1,10 @@
+# Python virtual environments
+.venv/
+venv/
+
+# MikroTik backup output
+mikrotik/backups/
+mikrotik/output/
+
+# Claude Code local state
+.claude/

backup.yml

@@ -0,0 +1,8 @@
+---
+- name: Install borgmatic and deploy per-host config
+  hosts: all
+  become: true
+  tags: never,backup
+
+  roles:
+    - role: backup

group_vars/all/backup.yml

@@ -0,0 +1,25 @@
+---
+# Borg Controller — auto-creates a repo per host on a BorgWarehouse-backed server.
+# borgcontroller_username / borgcontroller_password come from Semaphore secrets.
+borgcontroller_url: https://borgcontroller.internet-master.cz
+
+# Per-host borgmatic config. Hosts not listed here are skipped by the `backup` role.
+# `storage_size_gb` is stripped before rendering and used to size the controller-side
+# repo. `repositories` is auto-filled from the controller — don't set it manually.
+# Other keys are passed through verbatim to borgmatic, see
+# https://torsion.org/borgmatic/docs/reference/configuration/
+backup_hosts:
+  testipaclient:
+    storage_size_gb: 10
+    source_directories:
+      - /home/jakub
+    keep_daily: 7
+    keep_weekly: 4
+    keep_monthly: 6
+  portainer1-jim.im.lab:
+    storage_size_gb: 5
+    source_directories:
+      - /data/compose
+    keep_daily: 7
+    keep_weekly: 4
+    keep_monthly: 6

group_vars/all/users.yml

@@ -0,0 +1,22 @@
+---
+# Canonical user list — consumed by both setup_linux.yml and
+# initial_install/roles/users.
+users:
+  - name: automation
+    shell: /bin/bash
+    sudo_nopasswd: true
+    ssh_keys:
+      - "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEx+ltCKNIEM7F4PzGLv22cIu7N0Fpn5gxwV02xq0GS9 automation@internet-master.cz"
+
+  - name: hellsoslocal
+    shell: /bin/bash
+    sudo_nopasswd: true
+    ssh_keys:
+      - "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAB+wC3T4/HSs1q5jf34sCqicSQOb05k+bxfmNMMKEGzRrGT3BfCG428F19OBIswvcuBPC0Q4TpPz84BkiATCx2o1JUH1xIOFcHzxxXbyzHAhjwto1wOr1DkwZWAvDPbdnJ39OsC0EdmrAHSXut93q4vzOsLlS34bOWP1THGY9nBKOHwJUQmS5tLw6dqbhKA886TrPXJDR9euEC+SYaytMyDUPYEa6dlDyRp77eII/uI/hf/6e+34wm9XyFyGMiMrQeO0u6Gq5NhsoBlhrCW3ds0To+DBZ/YKNzpzcN+uPKM1+r9nN8KwdwjRkQEwSdB4osz/UeGTiXB0jwb0+ftFthBFdOil86cd1OiAMmuKB/19QHv0NsVhs2JocP5JcrAgx8ktQzLIkOM4lt6Kt9rjHv1KNfdsZdiHqlOwrDv9B2Ei44qEUAsWlFSzEi7R3mOED4F04N3FeQ9TkrRRH6SE733t1Kum2VAz6wr4BSNyYxQOCnXoANy/JyoM5e5tQ7pht7tuxX78zhFlC7pAmVu0dnCQimSsIsXNlYGM7DQ7QMva8mKu49V3B7tU0ChghSRJUb2Sg0tkTAxzCR+WOOf8kAnkXNOV5vExQ3h5Xmb52A+az37Hyex379ryKqpffaf9RTx9pBagQf1XbUtA4KazuEi3fMmqCQjz+xFZJAZQ== hellsos@hellsos-PC"
+      - "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKhfQt1VNQo8EbIog4yjU5VEF3mTyMEC7o1Qe95X4JwG jan@rabcan.cz"
+
+  - name: jimlocal
+    shell: /bin/bash
+    sudo_nopasswd: true
+    ssh_keys:
+      - "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPFS4fsqMjMMu/Bi/884bw7yJBqvWusDRESvanH6Owco jakub@jimbuntu"

initial_install/collections/requirements.yml

@@ -0,0 +1,2 @@
+collections:
+  - name: freeipa.ansible_freeipa

initial_install/playbook.yml

@@ -0,0 +1,47 @@
+---
+- name: Baseline system setup
+  hosts: all
+  become: true
+
+  roles:
+    - role: baseline_sudo
+      tags: sudo
+
+    - role: users
+      tags: users
+
+# ==============================
+# FREEIPA / SSSD (optional)
+# ==============================
+
+- name: FreeIPA client setup
+  hosts: all
+  become: true
+  tags: never,sssd
+
+  roles:
+    - role: freeipa_client
+
+# ==============================
+# DOCKHAND (optional)
+# ==============================
+
+- name: Install dockhand
+  hosts: all
+  become: true
+  tags: never,dockhand_install
+
+  roles:
+    - role: dockhand
+
+# ==============================
+# SSH HARDENING (run last!)
+# ==============================
+
+- name: SSH hardening
+  hosts: all
+  become: true
+  tags: never,hardening
+
+  roles:
+    - role: ssh_hardening

initial_install/roles/baseline_sudo/tasks/main.yml

@@ -0,0 +1,23 @@
+---
+- name: Ensure sudo package is installed
+  ansible.builtin.package:
+    name: sudo
+    state: present
+
+- name: Ensure automation user has passwordless sudo
+  ansible.builtin.copy:
+    dest: /etc/sudoers.d/automation
+    owner: root
+    group: root
+    mode: '0440'
+    content: |
+      automation ALL=(ALL:ALL) NOPASSWD: ALL
+    validate: 'visudo -cf %s'
+
+- name: Ensure sudo binary has correct permissions
+  ansible.builtin.file:
+    path: /usr/bin/sudo
+    owner: root
+    group: root
+    mode: '4755'
+  when: ansible_facts.os_family in ["Debian", "RedHat"]

initial_install/roles/dockhand/handlers/main.yml

@@ -0,0 +1,9 @@
+---
+- name: Reload systemd
+  ansible.builtin.systemd:
+    daemon_reload: true
+
+- name: Restart dockhand
+  ansible.builtin.systemd:
+    name: dockhand
+    state: restarted

initial_install/roles/dockhand/tasks/main.yml

@@ -0,0 +1,46 @@
+---
+- name: Install Docker and Compose
+  ansible.builtin.package:
+    name:
+      - docker.io
+      - docker-compose-v2
+    state: present
+
+- name: Ensure Docker is running
+  ansible.builtin.systemd:
+    name: docker
+    enabled: true
+    state: started
+
+- name: Ensure /docker/dockhand exists
+  ansible.builtin.file:
+    path: /docker/dockhand
+    state: directory
+    owner: root
+    group: root
+    mode: '0755'
+
+- name: Deploy dockhand docker-compose.yml
+  ansible.builtin.template:
+    src: docker-compose.yml.j2
+    dest: /docker/dockhand/docker-compose.yml
+    owner: root
+    group: root
+    mode: '0644'
+  notify: Restart dockhand
+
+- name: Deploy dockhand systemd unit
+  ansible.builtin.template:
+    src: dockhand.service.j2
+    dest: /etc/systemd/system/dockhand.service
+    owner: root
+    group: root
+    mode: '0644'
+  notify: Reload systemd
+
+- name: Enable and start dockhand
+  ansible.builtin.systemd:
+    name: dockhand
+    enabled: true
+    state: started
+    daemon_reload: true

initial_install/roles/dockhand/templates/docker-compose.yml.j2

@@ -0,0 +1,14 @@
+# Managed by Ansible — do not edit by hand.
+services:
+  dockhand:
+    image: fnsys/dockhand:latest
+    container_name: dockhand
+    restart: unless-stopped
+    ports:
+      - "3000:3000"
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+      - dockhand_data:/app/data
+
+volumes:
+  dockhand_data:

initial_install/roles/dockhand/templates/dockhand.service.j2

@@ -0,0 +1,16 @@
+[Unit]
+Description=dockhand (docker compose stack)
+Requires=docker.service
+After=docker.service network-online.target
+Wants=network-online.target
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+WorkingDirectory=/docker/dockhand
+ExecStart=/usr/bin/docker compose up -d --remove-orphans
+ExecStop=/usr/bin/docker compose down
+TimeoutStartSec=300
+
+[Install]
+WantedBy=multi-user.target

initial_install/roles/freeipa_client/handlers/main.yml

@@ -0,0 +1,5 @@
+---
+- name: Restart SSSD
+  ansible.builtin.service:
+    name: sssd
+    state: restarted

initial_install/roles/freeipa_client/tasks/main.yml

@@ -0,0 +1,67 @@
+---
+- name: Install FreeIPA client packages
+  ansible.builtin.package:
+    name:
+      - freeipa-client
+      - sssd
+      - sssd-tools
+      - oddjob
+      - oddjob-mkhomedir
+    state: present
+
+- name: Set hostname FQDN
+  ansible.builtin.hostname:
+    name: "{{ inventory_hostname }}.im.lab"
+
+- name: Check if FreeIPA client is already configured
+  ansible.builtin.stat:
+    path: /etc/ipa/default.conf
+  register: ipa_client_conf
+
+- name: Enroll to FreeIPA
+  ansible.builtin.command:
+    argv:
+      - ipa-client-install
+      - --domain=im.lab
+      - --realm=IM.LAB
+      - --server=ipa.im.lab
+      - "--hostname={{ inventory_hostname }}.im.lab"
+      - --mkhomedir
+      - --principal=admin
+      - --password={{ ipa_admin_password }}
+      - --unattended
+      - --force-join
+  no_log: false
+  when: not ipa_client_conf.stat.exists
+
+- name: Prioritize SSS over local accounts in NSS
+  ansible.builtin.lineinfile:
+    path: /etc/nsswitch.conf
+    regexp: '^{{ item }}:'
+    line: '{{ item }}: sss files systemd'
+  loop:
+    - passwd
+    - group
+  notify: Restart SSSD
+
+- name: Enable mkhomedir
+  ansible.builtin.command:
+    argv:
+      - authselect
+      - enable-feature
+      - with-mkhomedir
+  register: authselect_mkhomedir
+  changed_when: "'already enabled' not in authselect_mkhomedir.stdout"
+  failed_when: false
+
+- name: Enable and start oddjobd
+  ansible.builtin.service:
+    name: oddjobd
+    state: started
+    enabled: true
+
+- name: Enable and start SSSD
+  ansible.builtin.service:
+    name: sssd
+    state: started
+    enabled: true

initial_install/roles/ssh_hardening/handlers/main.yml

@@ -0,0 +1,8 @@
+---
+- name: Restart SSH
+  ansible.builtin.service:
+    name: "{{ 'sshd'
+              if ansible_facts.os_family in
+              ['RedHat','Rocky','AlmaLinux','Fedora','OracleLinux','Suse']
+              else 'ssh' }}"
+    state: restarted

initial_install/roles/ssh_hardening/tasks/main.yml

@@ -0,0 +1,29 @@
+---
+- name: Detect if system is Proxmox
+  ansible.builtin.stat:
+    path: /usr/bin/pveversion
+  register: proxmox_check
+
+- name: Ensure sshd_config.d exists
+  ansible.builtin.file:
+    path: /etc/ssh/sshd_config.d
+    state: directory
+
+- name: Deploy SSH hardening config
+  ansible.builtin.copy:
+    dest: /etc/ssh/sshd_config.d/99-ansible-hardening.conf
+    mode: '0644'
+    content: |
+      PasswordAuthentication no
+      ChallengeResponseAuthentication no
+      PubkeyAuthentication yes
+      AuthenticationMethods publickey
+      UsePAM yes
+
+      {% if not proxmox_check.stat.exists %}
+      PermitRootLogin no
+      {% else %}
+      PermitRootLogin prohibit-password
+      {% endif %}
+    validate: 'sshd -t -f %s'
+  notify: Restart SSH

initial_install/roles/users/tasks/main.yml

@@ -0,0 +1,39 @@
+---
+# `users` comes from group_vars/all/users.yml
+- name: Ensure users exist
+  ansible.builtin.user:
+    name: "{{ item.name }}"
+    shell: "{{ item.shell }}"
+    create_home: true
+  loop: "{{ users }}"
+
+# --------------------------------------------------
+# Configure passwordless sudo safely
+# --------------------------------------------------
+- name: Configure passwordless sudo
+  ansible.builtin.copy:
+    dest: "/etc/sudoers.d/{{ item.name }}"
+    mode: '0440'
+    owner: root
+    group: root
+    content: |
+      {{ item.name }} ALL=(ALL:ALL) NOPASSWD: ALL
+    validate: 'visudo -cf %s'
+  loop: "{{ users }}"
+  when: item.sudo_nopasswd | default(false)
+
+# --------------------------------------------------
+# Install SSH keys
+# --------------------------------------------------
+- name: Install authorized SSH keys
+  ansible.builtin.authorized_key:
+    user: "{{ item.name }}"
+    key: "{{ item.ssh_keys | join('\n') }}"
+    exclusive: true
+  loop: "{{ users }}"
+
+# --------------------------------------------------
+# Reset connection so sudo rules take effect immediately
+# --------------------------------------------------
+- name: Reset SSH connection
+  meta: reset_connection

inv_linuxes

@@ -1,9 +1,20 @@
-[linux_servers]
+[linux_servers_jim]
 jimbuntu ansible_host=192.168.19.4
-jim_storage ansible_host=192.168.19.7
+portainernode2_jim ansible_host=192.168.19.8
+galera3 ansible_host=192.168.19.92
+galera2 ansible_host=192.168.19.91
+testipaclient ansible_host=192.168.19.98
+testclient ansible_host=192.168.19.115
+portainer1-jim.im.lab ansible_host=192.168.19.7
+
+[linux_servers_hellsos]
 portainer2_hellsos ansible_host=192.168.52.9
 portainernode_hellsos ansible_host=192.168.52.21
-portainernode2_jim ansible_host=192.168.19.8
+
+[linux_servers:children]
+linux_servers_jim
+linux_servers_hellsos
+
 
 [local]
 localhost ansible_connection=local

mikrotikbackup_clean.yml

@@ -41,20 +41,20 @@
         state: directory
         mode: "0755"
       delegate_to: localhost
-      tags: backup
+      tags: [backup, never]
 
     - name: Export router config
       community.routeros.command:
         commands: /export terse show-sensitive
       register: export_cfg
-      tags: backup
+      tags: [backup, never]
 
     - name: Save export locally
       ansible.builtin.copy:
         content: "{{ export_cfg.stdout[0] }}"
         dest: "{{ backup_dir }}/{{ router_name }}-{{ ts }}.rsc"
       delegate_to: localhost
-      tags: backup
+      tags: [backup, never]
 
     # ----------------------------
     # Upgrade (tag: upgrade)
@@ -63,46 +63,84 @@
       community.routeros.command:
         commands: /system package update check-for-updates
       register: update_check
-      tags: upgrade
+      tags: [upgrade, never]
+
+    - name: Normalize update output
+      set_fact:
+        _update_text: "{{ update_check.stdout[0] | replace('\r', '') }}"
+      tags: [upgrade, never]
+
+    # ⬇️  Add this to see exactly what RouterOS returns before parsing
+    - name: Debug raw update output
+      ansible.builtin.debug:
+        msg: "{{ _update_text }}"
+      tags: [upgrade, never]
 
     - name: Parse installed and latest versions
       set_fact:
-        installed_version: "{{ update_check.stdout[0] | regex_search('installed-version: ([\\d.]+)', '\\1') | first }}"
-        latest_version: "{{ update_check.stdout[0] | regex_search('latest-version: ([\\d.]+)', '\\1') | first }}"
-      tags: upgrade
+        installed_version: >-
+          {{
+            (_update_text | regex_findall('(?:installed|current)-version:[ ]*([0-9A-Za-z.]+)'))[0]
+            if (_update_text | regex_findall('(?:installed|current)-version:[ ]*([0-9A-Za-z.]+)'))
+            else 'unknown'
+          }}
+        latest_version: >-
+          {{
+            (_update_text | regex_findall('(?:latest|newest)-version:[ ]*([0-9A-Za-z.]+)'))[0]
+            if (_update_text | regex_findall('(?:latest|newest)-version:[ ]*([0-9A-Za-z.]+)'))
+            else 'unknown'
+          }}
+      tags: [upgrade, never]
+
+    - name: Fail if versions could not be parsed
+      ansible.builtin.fail:
+        msg: >
+          Could not parse versions from update output.
+          Raw text was: {{ _update_text }}
+      when: installed_version == 'unknown' or latest_version == 'unknown'
+      tags: [upgrade, never]
+
+    - name: Debug parsed versions
+      ansible.builtin.debug:
+        msg:
+          - "Installed: {{ installed_version }}"
+          - "Latest: {{ latest_version }}"
+      tags: [upgrade, never]
 
     - name: Skip upgrade if already on latest
       ansible.builtin.debug:
         msg: "Router {{ router_name }} is already on latest version {{ installed_version }}. Skipping upgrade."
       when: installed_version == latest_version
-      tags: upgrade
+      tags: [upgrade, never]
 
     - name: Trigger package download and install
       community.routeros.command:
         commands: /system package update install
       register: upgrade_result
       when: installed_version != latest_version
-      tags: upgrade
+      tags: [upgrade, never]
 
     - name: Wait for router to come back online after reboot
-      ansible.builtin.wait_for_connection:
-        delay: 180
-        timeout: 300
-        sleep: 10
+      community.routeros.command:
+        commands: /system resource print
+      register: reboot_wait
+      until: reboot_wait is not failed
+      retries: 30
+      delay: 15
       when:
         - installed_version != latest_version
         - upgrade_result is not failed
-      tags: upgrade
+      tags: [upgrade, never]
 
     - name: Confirm upgraded version
       community.routeros.command:
         commands: /system resource print
       register: post_upgrade_info
       when: installed_version != latest_version
-      tags: upgrade
+      tags: [upgrade, never]
 
     - name: Show post-upgrade RouterOS version
       ansible.builtin.debug:
         msg: "{{ post_upgrade_info.stdout[0] | regex_search('version: .+') }}"
       when: installed_version != latest_version
-      tags: upgrade
+      tags: [upgrade, never]

remove_ssh_key_playbook.yml

@@ -0,0 +1,6 @@
+- name: Remove SSH key
+  hosts: all
+  become: yes
+  roles:
+    - role: manage_ssh_keys
+      remove_user: true

roles/backup/defaults/main.yml

@@ -0,0 +1,10 @@
+---
+# Schedule for our own borgmatic.timer (overrides the package-shipped unit).
+# OnCalendar uses systemd.time(7) syntax. RandomizedDelaySec spreads load so
+# every host doesn't hit the borg server at the same instant.
+borgmatic_oncalendar: "*-*-* 03:00:00"
+borgmatic_randomized_delay_sec: 3h
+borgmatic_persistent: true
+
+# Extra flags passed to the borgmatic invocation in our borgmatic.service.
+borgmatic_verbosity_args: "--verbosity -1 --syslog-verbosity 1"

roles/backup/tasks/borgcontroller.yml

@@ -0,0 +1,131 @@
+---
+- name: Login to borg controller
+  ansible.builtin.uri:
+    url: "{{ borgcontroller_url }}/api/auth/login"
+    method: POST
+    body_format: json
+    body:
+      username: "{{ borgcontroller_username }}"
+      password: "{{ borgcontroller_password }}"
+    status_code: 200
+  delegate_to: localhost
+  become: false
+  register: _bc_login
+  no_log: true
+
+- name: Get borg server SSH endpoint
+  ansible.builtin.uri:
+    url: "{{ borgcontroller_url }}/api/config"
+    method: GET
+    headers:
+      Cookie: "{{ _bc_login.cookies_string }}"
+  delegate_to: localhost
+  become: false
+  register: _bc_config
+
+- name: List repositories
+  ansible.builtin.uri:
+    url: "{{ borgcontroller_url }}/api/repositories"
+    method: GET
+    headers:
+      Cookie: "{{ _bc_login.cookies_string }}"
+  delegate_to: localhost
+  become: false
+  register: _bc_repos
+
+- name: Find existing repository for this host
+  ansible.builtin.set_fact:
+    _bc_existing: >-
+      {{ _bc_repos.json | selectattr('alias', 'eq', inventory_hostname) | list }}
+
+- name: Create repository if missing
+  ansible.builtin.uri:
+    url: "{{ borgcontroller_url }}/api/repositories"
+    method: POST
+    body_format: json
+    body:
+      alias: "{{ inventory_hostname }}"
+      sshPublicKey: "{{ root_ssh.ssh_public_key }}"
+      storageSize: "{{ backup_hosts[inventory_hostname].storage_size_gb | int }}"
+    headers:
+      Cookie: "{{ _bc_login.cookies_string }}"
+    status_code: [200, 201]
+  delegate_to: localhost
+  become: false
+  when: _bc_existing | length == 0
+
+- name: Update repository SSH key if root's key changed
+  ansible.builtin.uri:
+    url: "{{ borgcontroller_url }}/api/repositories/{{ _bc_existing[0].id }}"
+    method: PATCH
+    body_format: json
+    body:
+      sshPublicKey: "{{ root_ssh.ssh_public_key }}"
+    headers:
+      Cookie: "{{ _bc_login.cookies_string }}"
+    status_code: 200
+  delegate_to: localhost
+  become: false
+  when:
+    - _bc_existing | length > 0
+    - _bc_existing[0].sshPublicKey != root_ssh.ssh_public_key
+
+- name: Re-list repositories after possible create/update
+  ansible.builtin.uri:
+    url: "{{ borgcontroller_url }}/api/repositories"
+    method: GET
+    headers:
+      Cookie: "{{ _bc_login.cookies_string }}"
+  delegate_to: localhost
+  become: false
+  register: _bc_repos_after
+
+- name: Resolve repository for this host
+  ansible.builtin.set_fact:
+    _bc_repo: >-
+      {{ (_bc_repos_after.json | selectattr('alias', 'eq', inventory_hostname) | list)[0] }}
+
+- name: Build borg SSH URI
+  ansible.builtin.set_fact:
+    borgcontroller_repo_uri: "ssh://{{ _bc_config.json.borgSshHost }}/./{{ _bc_repo.id }}"
+    _bc_borg_host: "{{ _bc_config.json.borgSshHost.split('@')[1].split(':')[0] }}"
+    _bc_borg_port: "{{ _bc_config.json.borgSshHost.split('@')[1].split(':')[1] | default('22') }}"
+
+- name: Ensure /root/.ssh exists
+  ansible.builtin.file:
+    path: /root/.ssh
+    state: directory
+    owner: root
+    group: root
+    mode: '0700'
+
+- name: Scan borg server SSH host key
+  ansible.builtin.command: ssh-keyscan -p {{ _bc_borg_port }} {{ _bc_borg_host }}
+  register: _bc_keyscan
+  changed_when: false
+  check_mode: false
+
+- name: Trust borg server SSH host key (root known_hosts)
+  ansible.builtin.lineinfile:
+    path: /root/.ssh/known_hosts
+    line: "{{ item }}"
+    create: true
+    owner: root
+    group: root
+    mode: '0600'
+  loop: "{{ _bc_keyscan.stdout_lines }}"
+  when:
+    - item | length > 0
+    - not item.startswith('#')
+
+- name: Initialize borg repository (no-op if already initialized)
+  ansible.builtin.command:
+    cmd: borg init --encryption=repokey-blake2 {{ borgcontroller_repo_uri }}
+  environment:
+    BORG_PASSPHRASE: "{{ borg_passphrase }}"
+  register: _borg_init
+  changed_when: _borg_init.rc == 0
+  failed_when:
+    - _borg_init.rc != 0
+    - "'already exists' not in (_borg_init.stderr | default(''))"
+  no_log: true

roles/backup/tasks/main.yml

@@ -0,0 +1,105 @@
+---
+- name: Skip hosts without backup config
+  ansible.builtin.debug:
+    msg: "No entry in backup_hosts for {{ inventory_hostname }}; skipping backup role."
+  when: inventory_hostname not in (backup_hosts | default({}))
+
+- name: Configure borgmatic
+  when: inventory_hostname in (backup_hosts | default({}))
+  block:
+
+    - name: Ensure borg_passphrase is set (Semaphore secret)
+      ansible.builtin.assert:
+        that:
+          - borg_passphrase is defined
+          - borg_passphrase | length > 0
+        fail_msg: "borg_passphrase must be defined (provided by Semaphore secrets)"
+
+    - name: Install borgmatic
+      ansible.builtin.package:
+        name: borgmatic
+        state: present
+
+    - name: Ensure /etc/borgmatic exists
+      ansible.builtin.file:
+        path: /etc/borgmatic
+        state: directory
+        owner: root
+        group: root
+        mode: '0750'
+
+    - name: Write borg passphrase file
+      ansible.builtin.copy:
+        dest: /etc/borgmatic/passphrase
+        content: "{{ borg_passphrase }}"
+        owner: root
+        group: root
+        mode: '0600'
+      no_log: true
+
+    - name: Ensure root has an SSH key for the borg server
+      ansible.builtin.user:
+        name: root
+        generate_ssh_key: true
+        ssh_key_type: ed25519
+        ssh_key_file: .ssh/id_ed25519
+        ssh_key_comment: "borgmatic@{{ inventory_hostname }}"
+      register: root_ssh
+
+    - name: Register / look up repository on borg controller
+      ansible.builtin.include_tasks: borgcontroller.yml
+      when:
+        - borgcontroller_username is defined
+        - borgcontroller_password is defined
+
+    - name: Build borgmatic config (strip controller-only keys, inject repository + passcommand)
+      ansible.builtin.set_fact:
+        _borgmatic_config: >-
+          {{
+            (backup_hosts[inventory_hostname]
+             | dict2items
+             | rejectattr('key', 'in', ['storage_size_gb'])
+             | items2dict)
+            | combine({'encryption_passcommand': 'cat /etc/borgmatic/passphrase'})
+            | combine(
+                {'repositories': [{'path': borgcontroller_repo_uri, 'label': inventory_hostname}]}
+                if borgcontroller_repo_uri is defined else {}
+              )
+          }}
+
+    - name: Deploy borgmatic config
+      ansible.builtin.template:
+        src: borgmatic.yaml.j2
+        dest: /etc/borgmatic/config.yaml
+        owner: root
+        group: root
+        mode: '0640'
+
+    - name: Deploy borgmatic systemd service (overrides package unit)
+      ansible.builtin.template:
+        src: borgmatic.service.j2
+        dest: /etc/systemd/system/borgmatic.service
+        owner: root
+        group: root
+        mode: '0644'
+      register: _borgmatic_service_unit
+
+    - name: Deploy borgmatic systemd timer (overrides package unit)
+      ansible.builtin.template:
+        src: borgmatic.timer.j2
+        dest: /etc/systemd/system/borgmatic.timer
+        owner: root
+        group: root
+        mode: '0644'
+      register: _borgmatic_timer_unit
+
+    - name: Reload systemd if units changed
+      ansible.builtin.systemd:
+        daemon_reload: true
+      when: _borgmatic_service_unit is changed or _borgmatic_timer_unit is changed
+
+    - name: Enable and start borgmatic timer
+      ansible.builtin.systemd:
+        name: borgmatic.timer
+        enabled: true
+        state: started

roles/backup/templates/borgmatic.service.j2

@@ -0,0 +1,21 @@
+# Managed by Ansible — do not edit by hand.
+[Unit]
+Description=borgmatic backup
+Wants=network-online.target
+After=network-online.target
+# Don't run on battery power.
+ConditionACPower=true
+
+[Service]
+Type=oneshot
+# Lower priority so backups don't starve foreground work.
+Nice=19
+CPUSchedulingPolicy=batch
+IOSchedulingClass=best-effort
+IOSchedulingPriority=7
+IOWeight=100
+Restart=no
+# Prevent rate limiting of borgmatic log events.
+LogRateLimitIntervalSec=0
+# Delay start by a random amount handled in the timer; keep the service simple.
+ExecStart=systemd-inhibit --who="borgmatic" --what="sleep:shutdown" --why="Prevent interrupting scheduled backup" /usr/bin/borgmatic {{ borgmatic_verbosity_args }}

roles/backup/templates/borgmatic.timer.j2

@@ -0,0 +1,11 @@
+# Managed by Ansible — do not edit by hand.
+[Unit]
+Description=Run borgmatic backup
+
+[Timer]
+OnCalendar={{ borgmatic_oncalendar }}
+RandomizedDelaySec={{ borgmatic_randomized_delay_sec }}
+Persistent={{ borgmatic_persistent | bool | lower }}
+
+[Install]
+WantedBy=timers.target

roles/backup/templates/borgmatic.yaml.j2

@@ -0,0 +1,3 @@
+#jinja2: trim_blocks: True, lstrip_blocks: True
+# Managed by Ansible — do not edit by hand.
+{{ _borgmatic_config | to_nice_yaml(indent=2, width=1000) }}

roles/manage_ssh_keys/tasks/add_ssh_key.yml

@@ -0,0 +1,4 @@
+- name: Add user and authorized key
+  authorized_keys:
+    user: "{{ user }}"
+    key: "{{ key }}"

roles/manage_ssh_keys/tasks/main.yml

@@ -0,0 +1,5 @@
+- include_tasks: add_ssh_key.yml
+  when: add_user | default(false)
+
+- include_tasks: remove_ssh_key.yml
+  when: remove_user | default(false)

roles/manage_ssh_keys/tasks/remove_ssh_key.yml

@@ -0,0 +1,10 @@
+- name: Remove authorized key
+  authorized_keys:
+    user: "{{ user }}"
+    key: "{{ key }}"
+    state: absent
+
+- name: Ensure user is absent
+  user:
+    name: "{{ user }}"
+    state: absent

setup_linux.yml

@@ -2,29 +2,7 @@
 - name: Baseline user setup
   hosts: all
   become: true
-
-  vars:
-    users:
-      - name: automation
-        shell: /bin/bash
-        groups: []
-        sudo_nopasswd: true
-        ssh_keys:
-          - "ssh-ed25519 AAAAC3..."
-
-      - name: hellsos
-        shell: /bin/bash
-        groups: []
-        sudo_nopasswd: true
-        ssh_keys:
-          - "ssh-ed25519 AAAAC3..."
-
-      - name: jim
-        shell: /bin/bash
-        groups: []
-        sudo_nopasswd: true
-        ssh_keys:
-          - "ssh-ed25519 AAAAC3..."
+  tags: users
 
   tasks:
 
@@ -108,4 +86,37 @@
                   if ansible_facts.os_family in
                   ['RedHat','Rocky','AlmaLinux','Fedora','OracleLinux','Suse']
                   else 'ssh' }}"
-        state: restarted
+        state: restarted
+
+# ==============================
+# THIRD PLAY: HOSTNAME
+# ==============================
+
+- name: Set hostname from inventory
+  hosts: all
+  become: true
+  tags: never,hostname
+
+  tasks:
+
+    - name: Set system hostname to inventory_hostname
+      ansible.builtin.hostname:
+        name: "{{ inventory_hostname }}"
+
+# ==============================
+# FOURTH PLAY: FREEIPA / SSSD
+# ==============================
+
+- name: FreeIPA client setup
+  hosts: all
+  become: true
+  tags: never,sssd
+
+  roles:
+    - role: initial_install/roles/freeipa_client
+
+# ==============================
+# FIFTH PLAY: BACKUP
+# ==============================
+
+- import_playbook: backup.yml

test_sms.yml

@@ -7,7 +7,7 @@
     sms_username: "mikrotik"
     sms_password_send: "jdkotzHJIOPWhjtr32D"
     sms_password_recv: "jdkotzHJIOPWhjtr32D"
-    sms_wait_seconds: 120   # Wait 2 minutes for delivery
+    sms_wait_seconds: 15   # Wait 15s for delivery
 
   tasks:
     - name: Generate random test string