Manage own borgmatic systemd service and timer

Ship borgmatic.service and borgmatic.timer from the backup role instead
of relying on the package-provided units. The units are deployed to
/etc/systemd/system (overriding the package units), with a configurable
schedule via role defaults.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

.gitignore

@@ -0,0 +1,10 @@
+# Python virtual environments
+.venv/
+venv/
+
+# MikroTik backup output
+mikrotik/backups/
+mikrotik/output/
+
+# Claude Code local state
+.claude/

backup.yml

@@ -0,0 +1,8 @@
+---
+- name: Install borgmatic and deploy per-host config
+  hosts: all
+  become: true
+  tags: never,backup
+
+  roles:
+    - role: backup

group_vars/all/backup.yml

@@ -0,0 +1,25 @@
+---
+# Borg Controller — auto-creates a repo per host on a BorgWarehouse-backed server.
+# borgcontroller_username / borgcontroller_password come from Semaphore secrets.
+borgcontroller_url: https://borgcontroller.internet-master.cz
+
+# Per-host borgmatic config. Hosts not listed here are skipped by the `backup` role.
+# `storage_size_gb` is stripped before rendering and used to size the controller-side
+# repo. `repositories` is auto-filled from the controller — don't set it manually.
+# Other keys are passed through verbatim to borgmatic, see
+# https://torsion.org/borgmatic/docs/reference/configuration/
+backup_hosts:
+  testipaclient:
+    storage_size_gb: 10
+    source_directories:
+      - /home/jakub
+    keep_daily: 7
+    keep_weekly: 4
+    keep_monthly: 6
+  portainer1-jim.im.lab:
+    storage_size_gb: 5
+    source_directories:
+      - /data/compose
+    keep_daily: 7
+    keep_weekly: 4
+    keep_monthly: 6

group_vars/all/users.yml

@@ -0,0 +1,22 @@
+---
+# Canonical user list — consumed by both setup_linux.yml and
+# initial_install/roles/users.
+users:
+  - name: automation
+    shell: /bin/bash
+    sudo_nopasswd: true
+    ssh_keys:
+      - "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEx+ltCKNIEM7F4PzGLv22cIu7N0Fpn5gxwV02xq0GS9 automation@internet-master.cz"
+
+  - name: hellsoslocal
+    shell: /bin/bash
+    sudo_nopasswd: true
+    ssh_keys:
+      - "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAB+wC3T4/HSs1q5jf34sCqicSQOb05k+bxfmNMMKEGzRrGT3BfCG428F19OBIswvcuBPC0Q4TpPz84BkiATCx2o1JUH1xIOFcHzxxXbyzHAhjwto1wOr1DkwZWAvDPbdnJ39OsC0EdmrAHSXut93q4vzOsLlS34bOWP1THGY9nBKOHwJUQmS5tLw6dqbhKA886TrPXJDR9euEC+SYaytMyDUPYEa6dlDyRp77eII/uI/hf/6e+34wm9XyFyGMiMrQeO0u6Gq5NhsoBlhrCW3ds0To+DBZ/YKNzpzcN+uPKM1+r9nN8KwdwjRkQEwSdB4osz/UeGTiXB0jwb0+ftFthBFdOil86cd1OiAMmuKB/19QHv0NsVhs2JocP5JcrAgx8ktQzLIkOM4lt6Kt9rjHv1KNfdsZdiHqlOwrDv9B2Ei44qEUAsWlFSzEi7R3mOED4F04N3FeQ9TkrRRH6SE733t1Kum2VAz6wr4BSNyYxQOCnXoANy/JyoM5e5tQ7pht7tuxX78zhFlC7pAmVu0dnCQimSsIsXNlYGM7DQ7QMva8mKu49V3B7tU0ChghSRJUb2Sg0tkTAxzCR+WOOf8kAnkXNOV5vExQ3h5Xmb52A+az37Hyex379ryKqpffaf9RTx9pBagQf1XbUtA4KazuEi3fMmqCQjz+xFZJAZQ== hellsos@hellsos-PC"
+      - "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKhfQt1VNQo8EbIog4yjU5VEF3mTyMEC7o1Qe95X4JwG jan@rabcan.cz"
+
+  - name: jimlocal
+    shell: /bin/bash
+    sudo_nopasswd: true
+    ssh_keys:
+      - "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPFS4fsqMjMMu/Bi/884bw7yJBqvWusDRESvanH6Owco jakub@jimbuntu"

homarr.yml

@@ -0,0 +1,39 @@
+- name: Update Homarr
+  hosts: linux_servers
+  become: true
+  gather_facts: false
+
+  vars:
+    homarr_project: homarr
+    homarr_compose_file: /data/compose/homarr/docker-compose-homarr.yml
+    homarr_service: homarr
+    homarr_port: 7575
+
+  tasks:
+    - name: Pull latest Homarr image
+      community.docker.docker_compose_v2:
+        project_src: "{{ homarr_compose_file | dirname }}"
+        files:
+          - "{{ homarr_compose_file | basename }}"
+        pull: always
+
+    - name: Recreate Homarr service
+      community.docker.docker_compose_v2:
+        project_src: "{{ homarr_compose_file | dirname }}"
+        files:
+          - "{{ homarr_compose_file | basename }}"
+        services:
+          - "{{ homarr_service }}"
+        state: present
+        recreate: always
+
+    - name: Wait for Homarr port
+      ansible.builtin.wait_for:
+        host: 127.0.0.1
+        port: "{{ homarr_port }}"
+        timeout: 60
+
+    - name: Check Homarr HTTP endpoint
+      ansible.builtin.uri:
+        url: "http://127.0.0.1:{{ homarr_port }}/"
+        status_code: 200

initial_install/collections/requirements.yml

@@ -0,0 +1,2 @@
+collections:
+  - name: freeipa.ansible_freeipa

initial_install/playbook.yml

@@ -0,0 +1,47 @@
+---
+- name: Baseline system setup
+  hosts: all
+  become: true
+
+  roles:
+    - role: baseline_sudo
+      tags: sudo
+
+    - role: users
+      tags: users
+
+# ==============================
+# FREEIPA / SSSD (optional)
+# ==============================
+
+- name: FreeIPA client setup
+  hosts: all
+  become: true
+  tags: never,sssd
+
+  roles:
+    - role: freeipa_client
+
+# ==============================
+# DOCKHAND (optional)
+# ==============================
+
+- name: Install dockhand
+  hosts: all
+  become: true
+  tags: never,dockhand_install
+
+  roles:
+    - role: dockhand
+
+# ==============================
+# SSH HARDENING (run last!)
+# ==============================
+
+- name: SSH hardening
+  hosts: all
+  become: true
+  tags: never,hardening
+
+  roles:
+    - role: ssh_hardening

initial_install/roles/baseline_sudo/tasks/main.yml

@@ -0,0 +1,23 @@
+---
+- name: Ensure sudo package is installed
+  ansible.builtin.package:
+    name: sudo
+    state: present
+
+- name: Ensure automation user has passwordless sudo
+  ansible.builtin.copy:
+    dest: /etc/sudoers.d/automation
+    owner: root
+    group: root
+    mode: '0440'
+    content: |
+      automation ALL=(ALL:ALL) NOPASSWD: ALL
+    validate: 'visudo -cf %s'
+
+- name: Ensure sudo binary has correct permissions
+  ansible.builtin.file:
+    path: /usr/bin/sudo
+    owner: root
+    group: root
+    mode: '4755'
+  when: ansible_facts.os_family in ["Debian", "RedHat"]

initial_install/roles/dockhand/handlers/main.yml

@@ -0,0 +1,9 @@
+---
+- name: Reload systemd
+  ansible.builtin.systemd:
+    daemon_reload: true
+
+- name: Restart dockhand
+  ansible.builtin.systemd:
+    name: dockhand
+    state: restarted

initial_install/roles/dockhand/tasks/main.yml

@@ -0,0 +1,46 @@
+---
+- name: Install Docker and Compose
+  ansible.builtin.package:
+    name:
+      - docker.io
+      - docker-compose-v2
+    state: present
+
+- name: Ensure Docker is running
+  ansible.builtin.systemd:
+    name: docker
+    enabled: true
+    state: started
+
+- name: Ensure /docker/dockhand exists
+  ansible.builtin.file:
+    path: /docker/dockhand
+    state: directory
+    owner: root
+    group: root
+    mode: '0755'
+
+- name: Deploy dockhand docker-compose.yml
+  ansible.builtin.template:
+    src: docker-compose.yml.j2
+    dest: /docker/dockhand/docker-compose.yml
+    owner: root
+    group: root
+    mode: '0644'
+  notify: Restart dockhand
+
+- name: Deploy dockhand systemd unit
+  ansible.builtin.template:
+    src: dockhand.service.j2
+    dest: /etc/systemd/system/dockhand.service
+    owner: root
+    group: root
+    mode: '0644'
+  notify: Reload systemd
+
+- name: Enable and start dockhand
+  ansible.builtin.systemd:
+    name: dockhand
+    enabled: true
+    state: started
+    daemon_reload: true

initial_install/roles/dockhand/templates/docker-compose.yml.j2

@@ -0,0 +1,14 @@
+# Managed by Ansible — do not edit by hand.
+services:
+  dockhand:
+    image: fnsys/dockhand:latest
+    container_name: dockhand
+    restart: unless-stopped
+    ports:
+      - "3000:3000"
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+      - dockhand_data:/app/data
+
+volumes:
+  dockhand_data:

initial_install/roles/dockhand/templates/dockhand.service.j2

@@ -0,0 +1,16 @@
+[Unit]
+Description=dockhand (docker compose stack)
+Requires=docker.service
+After=docker.service network-online.target
+Wants=network-online.target
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+WorkingDirectory=/docker/dockhand
+ExecStart=/usr/bin/docker compose up -d --remove-orphans
+ExecStop=/usr/bin/docker compose down
+TimeoutStartSec=300
+
+[Install]
+WantedBy=multi-user.target

initial_install/roles/freeipa_client/handlers/main.yml

@@ -0,0 +1,5 @@
+---
+- name: Restart SSSD
+  ansible.builtin.service:
+    name: sssd
+    state: restarted

initial_install/roles/freeipa_client/tasks/main.yml

@@ -0,0 +1,67 @@
+---
+- name: Install FreeIPA client packages
+  ansible.builtin.package:
+    name:
+      - freeipa-client
+      - sssd
+      - sssd-tools
+      - oddjob
+      - oddjob-mkhomedir
+    state: present
+
+- name: Set hostname FQDN
+  ansible.builtin.hostname:
+    name: "{{ inventory_hostname }}.im.lab"
+
+- name: Check if FreeIPA client is already configured
+  ansible.builtin.stat:
+    path: /etc/ipa/default.conf
+  register: ipa_client_conf
+
+- name: Enroll to FreeIPA
+  ansible.builtin.command:
+    argv:
+      - ipa-client-install
+      - --domain=im.lab
+      - --realm=IM.LAB
+      - --server=ipa.im.lab
+      - "--hostname={{ inventory_hostname }}.im.lab"
+      - --mkhomedir
+      - --principal=admin
+      - --password={{ ipa_admin_password }}
+      - --unattended
+      - --force-join
+  no_log: false
+  when: not ipa_client_conf.stat.exists
+
+- name: Prioritize SSS over local accounts in NSS
+  ansible.builtin.lineinfile:
+    path: /etc/nsswitch.conf
+    regexp: '^{{ item }}:'
+    line: '{{ item }}: sss files systemd'
+  loop:
+    - passwd
+    - group
+  notify: Restart SSSD
+
+- name: Enable mkhomedir
+  ansible.builtin.command:
+    argv:
+      - authselect
+      - enable-feature
+      - with-mkhomedir
+  register: authselect_mkhomedir
+  changed_when: "'already enabled' not in authselect_mkhomedir.stdout"
+  failed_when: false
+
+- name: Enable and start oddjobd
+  ansible.builtin.service:
+    name: oddjobd
+    state: started
+    enabled: true
+
+- name: Enable and start SSSD
+  ansible.builtin.service:
+    name: sssd
+    state: started
+    enabled: true

initial_install/roles/ssh_hardening/handlers/main.yml

@@ -0,0 +1,8 @@
+---
+- name: Restart SSH
+  ansible.builtin.service:
+    name: "{{ 'sshd'
+              if ansible_facts.os_family in
+              ['RedHat','Rocky','AlmaLinux','Fedora','OracleLinux','Suse']
+              else 'ssh' }}"
+    state: restarted

initial_install/roles/ssh_hardening/tasks/main.yml

@@ -0,0 +1,29 @@
+---
+- name: Detect if system is Proxmox
+  ansible.builtin.stat:
+    path: /usr/bin/pveversion
+  register: proxmox_check
+
+- name: Ensure sshd_config.d exists
+  ansible.builtin.file:
+    path: /etc/ssh/sshd_config.d
+    state: directory
+
+- name: Deploy SSH hardening config
+  ansible.builtin.copy:
+    dest: /etc/ssh/sshd_config.d/99-ansible-hardening.conf
+    mode: '0644'
+    content: |
+      PasswordAuthentication no
+      ChallengeResponseAuthentication no
+      PubkeyAuthentication yes
+      AuthenticationMethods publickey
+      UsePAM yes
+
+      {% if not proxmox_check.stat.exists %}
+      PermitRootLogin no
+      {% else %}
+      PermitRootLogin prohibit-password
+      {% endif %}
+    validate: 'sshd -t -f %s'
+  notify: Restart SSH

initial_install/roles/users/tasks/main.yml

@@ -0,0 +1,39 @@
+---
+# `users` comes from group_vars/all/users.yml
+- name: Ensure users exist
+  ansible.builtin.user:
+    name: "{{ item.name }}"
+    shell: "{{ item.shell }}"
+    create_home: true
+  loop: "{{ users }}"
+
+# --------------------------------------------------
+# Configure passwordless sudo safely
+# --------------------------------------------------
+- name: Configure passwordless sudo
+  ansible.builtin.copy:
+    dest: "/etc/sudoers.d/{{ item.name }}"
+    mode: '0440'
+    owner: root
+    group: root
+    content: |
+      {{ item.name }} ALL=(ALL:ALL) NOPASSWD: ALL
+    validate: 'visudo -cf %s'
+  loop: "{{ users }}"
+  when: item.sudo_nopasswd | default(false)
+
+# --------------------------------------------------
+# Install SSH keys
+# --------------------------------------------------
+- name: Install authorized SSH keys
+  ansible.builtin.authorized_key:
+    user: "{{ item.name }}"
+    key: "{{ item.ssh_keys | join('\n') }}"
+    exclusive: true
+  loop: "{{ users }}"
+
+# --------------------------------------------------
+# Reset connection so sudo rules take effect immediately
+# --------------------------------------------------
+- name: Reset SSH connection
+  meta: reset_connection

inv_linuxes

@@ -1,5 +1,20 @@
-[linux_servers]
+[linux_servers_jim]
 jimbuntu ansible_host=192.168.19.4
-jim_storage ansible_host=192.168.19.7
-portainer2 ansible_host=192.168.52.9
-portainernode ansible_host=192.168.52.21
+portainernode2_jim ansible_host=192.168.19.8
+galera3 ansible_host=192.168.19.92
+galera2 ansible_host=192.168.19.91
+testipaclient ansible_host=192.168.19.98
+testclient ansible_host=192.168.19.115
+portainer1-jim.im.lab ansible_host=192.168.19.7
+
+[linux_servers_hellsos]
+portainer2_hellsos ansible_host=192.168.52.9
+portainernode_hellsos ansible_host=192.168.52.21
+
+[linux_servers:children]
+linux_servers_jim
+linux_servers_hellsos
+
+
+[local]
+localhost ansible_connection=local

inv_mikrotiks

@@ -1,2 +1,16 @@
-[mikrotiks]
-storage ansible_host=192.168.19.3 ansible_ssh_common_args='-o StrictHostKeyChecking=no'
+[mikrotik_routers]
+jim_main ansible_host=192.168.19.2
+jim_gw2 ansible_host=192.168.19.3
+hellsos ansible_host=192.168.40.1
+ewolet ansible_host=192.168.90.1
+Poli ansible_host=192.168.2.1
+Schmid ansible_host=192.168.177.1
+#Volf ansible_host=192.168.88.1
+fencl_home ansible_host=192.168.68.1
+fencl_tata ansible_host=192.168.69.1
+
+
+[mikrotik_routers:vars]
+ansible_connection=network_cli
+ansible_network_os=community.routeros.routeros
+ansible_command_timeout=15

mikrotikbackup.yml

@@ -7,7 +7,7 @@
 
     - name: Ensure output directory exists
       ansible.builtin.file:
-        path: output
+        path: /opt/mikrotik_backups
         state: directory
         mode: '0755'
       delegate_to: localhost
@@ -33,7 +33,7 @@
         current_date: "{{ date_output.stdout }}"
 
     - name: Export router config
-      shell: timeout 15 ssh -o StrictHostKeyChecking=no {{ ansible_user }}@{{ ansible_host }} -p {{ ansible_port }} "/export"
+      shell: timeout 15 ssh -o StrictHostKeyChecking=no {{ ansible_user }}@{{ ansible_host }} -p {{ ansible_port }} "/export show-sensitive"
       register: export_output
       delegate_to: localhost
       when: system_identity.rc == 0
@@ -42,7 +42,7 @@
     - name: Save export to local file
       ansible.builtin.copy:
         content: "{{ export_output.stdout }}"
-        dest: "output/{{ router_name }}-{{ current_date }}.config"
+        dest: "/opt/mikrotik_backups/{{ router_name }}-{{ current_date }}.config"
       delegate_to: localhost
       when: export_output.rc == 0
 
@@ -52,7 +52,7 @@
       when: system_identity.rc == 0
 
     - name: Download binary backup
-      shell: timeout 15 scp -o StrictHostKeyChecking=no -P {{ ansible_port }} {{ ansible_user }}@{{ ansible_host }}:{{ router_name }}-{{ current_date }}-backup.backup output/
+      shell: timeout 15 scp -o StrictHostKeyChecking=no -P {{ ansible_port }} {{ ansible_user }}@{{ ansible_host }}:{{ router_name }}-{{ current_date }}-backup.backup /opt/mikrotik_backups/
       delegate_to: localhost
       when: system_identity.rc == 0
 

mikrotikbackup_clean.yml

@@ -0,0 +1,146 @@
+- name: Backup and/or Upgrade MikroTik
+  hosts: mikrotik_routers
+  gather_facts: no
+
+  vars:
+    backup_dir: /opt/mikrotik_backups/
+
+  tasks:
+
+    # ----------------------------
+    # Always: identity + timestamp
+    # ----------------------------
+    - name: Get router identity
+      community.routeros.command:
+        commands: /system identity print
+      register: identity_raw
+      tags: always
+
+    - name: Parse router name
+      set_fact:
+        router_name: "{{ identity_raw.stdout[0].split(': ')[1] | trim }}"
+      tags: always
+
+    - name: Get timestamp
+      ansible.builtin.command: date +%Y-%m-%d_%H-%M-%S
+      register: date_out
+      delegate_to: localhost
+      tags: always
+
+    - name: Set timestamp fact
+      set_fact:
+        ts: "{{ date_out.stdout }}"
+      tags: always
+
+    # ----------------------------
+    # Backup (tag: backup)
+    # ----------------------------
+    - name: Ensure local backup directory exists
+      ansible.builtin.file:
+        path: "{{ backup_dir }}"
+        state: directory
+        mode: "0755"
+      delegate_to: localhost
+      tags: [backup, never]
+
+    - name: Export router config
+      community.routeros.command:
+        commands: /export terse show-sensitive
+      register: export_cfg
+      tags: [backup, never]
+
+    - name: Save export locally
+      ansible.builtin.copy:
+        content: "{{ export_cfg.stdout[0] }}"
+        dest: "{{ backup_dir }}/{{ router_name }}-{{ ts }}.rsc"
+      delegate_to: localhost
+      tags: [backup, never]
+
+    # ----------------------------
+    # Upgrade (tag: upgrade)
+    # ----------------------------
+    - name: Check current and latest available package versions
+      community.routeros.command:
+        commands: /system package update check-for-updates
+      register: update_check
+      tags: [upgrade, never]
+
+    - name: Normalize update output
+      set_fact:
+        _update_text: "{{ update_check.stdout[0] | replace('\r', '') }}"
+      tags: [upgrade, never]
+
+    # ⬇️  Add this to see exactly what RouterOS returns before parsing
+    - name: Debug raw update output
+      ansible.builtin.debug:
+        msg: "{{ _update_text }}"
+      tags: [upgrade, never]
+
+    - name: Parse installed and latest versions
+      set_fact:
+        installed_version: >-
+          {{
+            (_update_text | regex_findall('(?:installed|current)-version:[ ]*([0-9A-Za-z.]+)'))[0]
+            if (_update_text | regex_findall('(?:installed|current)-version:[ ]*([0-9A-Za-z.]+)'))
+            else 'unknown'
+          }}
+        latest_version: >-
+          {{
+            (_update_text | regex_findall('(?:latest|newest)-version:[ ]*([0-9A-Za-z.]+)'))[0]
+            if (_update_text | regex_findall('(?:latest|newest)-version:[ ]*([0-9A-Za-z.]+)'))
+            else 'unknown'
+          }}
+      tags: [upgrade, never]
+
+    - name: Fail if versions could not be parsed
+      ansible.builtin.fail:
+        msg: >
+          Could not parse versions from update output.
+          Raw text was: {{ _update_text }}
+      when: installed_version == 'unknown' or latest_version == 'unknown'
+      tags: [upgrade, never]
+
+    - name: Debug parsed versions
+      ansible.builtin.debug:
+        msg:
+          - "Installed: {{ installed_version }}"
+          - "Latest: {{ latest_version }}"
+      tags: [upgrade, never]
+
+    - name: Skip upgrade if already on latest
+      ansible.builtin.debug:
+        msg: "Router {{ router_name }} is already on latest version {{ installed_version }}. Skipping upgrade."
+      when: installed_version == latest_version
+      tags: [upgrade, never]
+
+    - name: Trigger package download and install
+      community.routeros.command:
+        commands: /system package update install
+      register: upgrade_result
+      when: installed_version != latest_version
+      tags: [upgrade, never]
+
+    - name: Wait for router to come back online after reboot
+      community.routeros.command:
+        commands: /system resource print
+      register: reboot_wait
+      until: reboot_wait is not failed
+      retries: 30
+      delay: 15
+      when:
+        - installed_version != latest_version
+        - upgrade_result is not failed
+      tags: [upgrade, never]
+
+    - name: Confirm upgraded version
+      community.routeros.command:
+        commands: /system resource print
+      register: post_upgrade_info
+      when: installed_version != latest_version
+      tags: [upgrade, never]
+
+    - name: Show post-upgrade RouterOS version
+      ansible.builtin.debug:
+        msg: "{{ post_upgrade_info.stdout[0] | regex_search('version: .+') }}"
+      when: installed_version != latest_version
+      tags: [upgrade, never]

remove_ssh_key_playbook.yml

@@ -0,0 +1,6 @@
+- name: Remove SSH key
+  hosts: all
+  become: yes
+  roles:
+    - role: manage_ssh_keys
+      remove_user: true

requirements.yml

@@ -0,0 +1,2 @@
+collections:
+  - name: community.routeros

roles/backup/defaults/main.yml

@@ -0,0 +1,10 @@
+---
+# Schedule for our own borgmatic.timer (overrides the package-shipped unit).
+# OnCalendar uses systemd.time(7) syntax. RandomizedDelaySec spreads load so
+# every host doesn't hit the borg server at the same instant.
+borgmatic_oncalendar: "*-*-* 03:00:00"
+borgmatic_randomized_delay_sec: 3h
+borgmatic_persistent: true
+
+# Extra flags passed to the borgmatic invocation in our borgmatic.service.
+borgmatic_verbosity_args: "--verbosity -1 --syslog-verbosity 1"

roles/backup/tasks/borgcontroller.yml

@@ -0,0 +1,131 @@
+---
+- name: Login to borg controller
+  ansible.builtin.uri:
+    url: "{{ borgcontroller_url }}/api/auth/login"
+    method: POST
+    body_format: json
+    body:
+      username: "{{ borgcontroller_username }}"
+      password: "{{ borgcontroller_password }}"
+    status_code: 200
+  delegate_to: localhost
+  become: false
+  register: _bc_login
+  no_log: true
+
+- name: Get borg server SSH endpoint
+  ansible.builtin.uri:
+    url: "{{ borgcontroller_url }}/api/config"
+    method: GET
+    headers:
+      Cookie: "{{ _bc_login.cookies_string }}"
+  delegate_to: localhost
+  become: false
+  register: _bc_config
+
+- name: List repositories
+  ansible.builtin.uri:
+    url: "{{ borgcontroller_url }}/api/repositories"
+    method: GET
+    headers:
+      Cookie: "{{ _bc_login.cookies_string }}"
+  delegate_to: localhost
+  become: false
+  register: _bc_repos
+
+- name: Find existing repository for this host
+  ansible.builtin.set_fact:
+    _bc_existing: >-
+      {{ _bc_repos.json | selectattr('alias', 'eq', inventory_hostname) | list }}
+
+- name: Create repository if missing
+  ansible.builtin.uri:
+    url: "{{ borgcontroller_url }}/api/repositories"
+    method: POST
+    body_format: json
+    body:
+      alias: "{{ inventory_hostname }}"
+      sshPublicKey: "{{ root_ssh.ssh_public_key }}"
+      storageSize: "{{ backup_hosts[inventory_hostname].storage_size_gb | int }}"
+    headers:
+      Cookie: "{{ _bc_login.cookies_string }}"
+    status_code: [200, 201]
+  delegate_to: localhost
+  become: false
+  when: _bc_existing | length == 0
+
+- name: Update repository SSH key if root's key changed
+  ansible.builtin.uri:
+    url: "{{ borgcontroller_url }}/api/repositories/{{ _bc_existing[0].id }}"
+    method: PATCH
+    body_format: json
+    body:
+      sshPublicKey: "{{ root_ssh.ssh_public_key }}"
+    headers:
+      Cookie: "{{ _bc_login.cookies_string }}"
+    status_code: 200
+  delegate_to: localhost
+  become: false
+  when:
+    - _bc_existing | length > 0
+    - _bc_existing[0].sshPublicKey != root_ssh.ssh_public_key
+
+- name: Re-list repositories after possible create/update
+  ansible.builtin.uri:
+    url: "{{ borgcontroller_url }}/api/repositories"
+    method: GET
+    headers:
+      Cookie: "{{ _bc_login.cookies_string }}"
+  delegate_to: localhost
+  become: false
+  register: _bc_repos_after
+
+- name: Resolve repository for this host
+  ansible.builtin.set_fact:
+    _bc_repo: >-
+      {{ (_bc_repos_after.json | selectattr('alias', 'eq', inventory_hostname) | list)[0] }}
+
+- name: Build borg SSH URI
+  ansible.builtin.set_fact:
+    borgcontroller_repo_uri: "ssh://{{ _bc_config.json.borgSshHost }}/./{{ _bc_repo.id }}"
+    _bc_borg_host: "{{ _bc_config.json.borgSshHost.split('@')[1].split(':')[0] }}"
+    _bc_borg_port: "{{ _bc_config.json.borgSshHost.split('@')[1].split(':')[1] | default('22') }}"
+
+- name: Ensure /root/.ssh exists
+  ansible.builtin.file:
+    path: /root/.ssh
+    state: directory
+    owner: root
+    group: root
+    mode: '0700'
+
+- name: Scan borg server SSH host key
+  ansible.builtin.command: ssh-keyscan -p {{ _bc_borg_port }} {{ _bc_borg_host }}
+  register: _bc_keyscan
+  changed_when: false
+  check_mode: false
+
+- name: Trust borg server SSH host key (root known_hosts)
+  ansible.builtin.lineinfile:
+    path: /root/.ssh/known_hosts
+    line: "{{ item }}"
+    create: true
+    owner: root
+    group: root
+    mode: '0600'
+  loop: "{{ _bc_keyscan.stdout_lines }}"
+  when:
+    - item | length > 0
+    - not item.startswith('#')
+
+- name: Initialize borg repository (no-op if already initialized)
+  ansible.builtin.command:
+    cmd: borg init --encryption=repokey-blake2 {{ borgcontroller_repo_uri }}
+  environment:
+    BORG_PASSPHRASE: "{{ borg_passphrase }}"
+  register: _borg_init
+  changed_when: _borg_init.rc == 0
+  failed_when:
+    - _borg_init.rc != 0
+    - "'already exists' not in (_borg_init.stderr | default(''))"
+  no_log: true

roles/backup/tasks/main.yml

@@ -0,0 +1,105 @@
+---
+- name: Skip hosts without backup config
+  ansible.builtin.debug:
+    msg: "No entry in backup_hosts for {{ inventory_hostname }}; skipping backup role."
+  when: inventory_hostname not in (backup_hosts | default({}))
+
+- name: Configure borgmatic
+  when: inventory_hostname in (backup_hosts | default({}))
+  block:
+
+    - name: Ensure borg_passphrase is set (Semaphore secret)
+      ansible.builtin.assert:
+        that:
+          - borg_passphrase is defined
+          - borg_passphrase | length > 0
+        fail_msg: "borg_passphrase must be defined (provided by Semaphore secrets)"
+
+    - name: Install borgmatic
+      ansible.builtin.package:
+        name: borgmatic
+        state: present
+
+    - name: Ensure /etc/borgmatic exists
+      ansible.builtin.file:
+        path: /etc/borgmatic
+        state: directory
+        owner: root
+        group: root
+        mode: '0750'
+
+    - name: Write borg passphrase file
+      ansible.builtin.copy:
+        dest: /etc/borgmatic/passphrase
+        content: "{{ borg_passphrase }}"
+        owner: root
+        group: root
+        mode: '0600'
+      no_log: true
+
+    - name: Ensure root has an SSH key for the borg server
+      ansible.builtin.user:
+        name: root
+        generate_ssh_key: true
+        ssh_key_type: ed25519
+        ssh_key_file: .ssh/id_ed25519
+        ssh_key_comment: "borgmatic@{{ inventory_hostname }}"
+      register: root_ssh
+
+    - name: Register / look up repository on borg controller
+      ansible.builtin.include_tasks: borgcontroller.yml
+      when:
+        - borgcontroller_username is defined
+        - borgcontroller_password is defined
+
+    - name: Build borgmatic config (strip controller-only keys, inject repository + passcommand)
+      ansible.builtin.set_fact:
+        _borgmatic_config: >-
+          {{
+            (backup_hosts[inventory_hostname]
+             | dict2items
+             | rejectattr('key', 'in', ['storage_size_gb'])
+             | items2dict)
+            | combine({'encryption_passcommand': 'cat /etc/borgmatic/passphrase'})
+            | combine(
+                {'repositories': [{'path': borgcontroller_repo_uri, 'label': inventory_hostname}]}
+                if borgcontroller_repo_uri is defined else {}
+              )
+          }}
+
+    - name: Deploy borgmatic config
+      ansible.builtin.template:
+        src: borgmatic.yaml.j2
+        dest: /etc/borgmatic/config.yaml
+        owner: root
+        group: root
+        mode: '0640'
+
+    - name: Deploy borgmatic systemd service (overrides package unit)
+      ansible.builtin.template:
+        src: borgmatic.service.j2
+        dest: /etc/systemd/system/borgmatic.service
+        owner: root
+        group: root
+        mode: '0644'
+      register: _borgmatic_service_unit
+
+    - name: Deploy borgmatic systemd timer (overrides package unit)
+      ansible.builtin.template:
+        src: borgmatic.timer.j2
+        dest: /etc/systemd/system/borgmatic.timer
+        owner: root
+        group: root
+        mode: '0644'
+      register: _borgmatic_timer_unit
+
+    - name: Reload systemd if units changed
+      ansible.builtin.systemd:
+        daemon_reload: true
+      when: _borgmatic_service_unit is changed or _borgmatic_timer_unit is changed
+
+    - name: Enable and start borgmatic timer
+      ansible.builtin.systemd:
+        name: borgmatic.timer
+        enabled: true
+        state: started

roles/backup/templates/borgmatic.service.j2

@@ -0,0 +1,21 @@
+# Managed by Ansible — do not edit by hand.
+[Unit]
+Description=borgmatic backup
+Wants=network-online.target
+After=network-online.target
+# Don't run on battery power.
+ConditionACPower=true
+
+[Service]
+Type=oneshot
+# Lower priority so backups don't starve foreground work.
+Nice=19
+CPUSchedulingPolicy=batch
+IOSchedulingClass=best-effort
+IOSchedulingPriority=7
+IOWeight=100
+Restart=no
+# Prevent rate limiting of borgmatic log events.
+LogRateLimitIntervalSec=0
+# Delay start by a random amount handled in the timer; keep the service simple.
+ExecStart=systemd-inhibit --who="borgmatic" --what="sleep:shutdown" --why="Prevent interrupting scheduled backup" /usr/bin/borgmatic {{ borgmatic_verbosity_args }}

roles/backup/templates/borgmatic.timer.j2

@@ -0,0 +1,11 @@
+# Managed by Ansible — do not edit by hand.
+[Unit]
+Description=Run borgmatic backup
+
+[Timer]
+OnCalendar={{ borgmatic_oncalendar }}
+RandomizedDelaySec={{ borgmatic_randomized_delay_sec }}
+Persistent={{ borgmatic_persistent | bool | lower }}
+
+[Install]
+WantedBy=timers.target

roles/backup/templates/borgmatic.yaml.j2

@@ -0,0 +1,3 @@
+#jinja2: trim_blocks: True, lstrip_blocks: True
+# Managed by Ansible — do not edit by hand.
+{{ _borgmatic_config | to_nice_yaml(indent=2, width=1000) }}

roles/manage_ssh_keys/tasks/add_ssh_key.yml

@@ -0,0 +1,4 @@
+- name: Add user and authorized key
+  authorized_keys:
+    user: "{{ user }}"
+    key: "{{ key }}"

roles/manage_ssh_keys/tasks/main.yml

@@ -0,0 +1,5 @@
+- include_tasks: add_ssh_key.yml
+  when: add_user | default(false)
+
+- include_tasks: remove_ssh_key.yml
+  when: remove_user | default(false)

roles/manage_ssh_keys/tasks/remove_ssh_key.yml

@@ -0,0 +1,10 @@
+- name: Remove authorized key
+  authorized_keys:
+    user: "{{ user }}"
+    key: "{{ key }}"
+    state: absent
+
+- name: Ensure user is absent
+  user:
+    name: "{{ user }}"
+    state: absent

setup_linux.yml

@@ -0,0 +1,122 @@
+---
+- name: Baseline user setup
+  hosts: all
+  become: true
+  tags: users
+
+  tasks:
+
+    - name: Pick sudo group per distro
+      set_fact:
+        sudo_group: >-
+          {{ 'wheel'
+             if ansible_facts.os_family in
+             ['RedHat','Rocky','AlmaLinux','Fedora','OracleLinux','Suse']
+             else 'sudo' }}
+
+    - name: Ensure user exists
+      ansible.builtin.user:
+        name: "{{ item.name }}"
+        shell: "{{ item.shell }}"
+        groups: "{{ sudo_group }}"
+        append: true
+        create_home: true
+      loop: "{{ users }}"
+
+    - name: Enforce authorized SSH keys
+      ansible.builtin.authorized_key:
+        user: "{{ item.name }}"
+        key: "{{ item.ssh_keys | join('\n') }}"
+        exclusive: true
+      loop: "{{ users }}"
+
+    - name: Grant passwordless sudo
+      ansible.builtin.copy:
+        dest: "/etc/sudoers.d/{{ item.name }}"
+        mode: '0440'
+        content: "{{ item.name }} ALL=(ALL) NOPASSWD:ALL\n"
+        validate: 'visudo -cf %s'
+      loop: "{{ users }}"
+      when: item.sudo_nopasswd
+
+# ==============================
+# SECOND PLAY: SSH HARDENING
+# ==============================
+
+- name: SSH Hardening
+  hosts: all
+  become: true
+  tags: never,hardening
+
+  tasks:
+
+    - name: Detect if system is Proxmox
+      ansible.builtin.stat:
+        path: /usr/bin/pveversion
+      register: proxmox_check
+
+    - name: Ensure sshd_config.d directory exists
+      ansible.builtin.file:
+        path: /etc/ssh/sshd_config.d
+        state: directory
+
+    - name: Deploy SSH hardening config
+      ansible.builtin.copy:
+        dest: /etc/ssh/sshd_config.d/99-ansible-hardening.conf
+        mode: '0644'
+        content: |
+          PasswordAuthentication no
+          ChallengeResponseAuthentication no
+          PubkeyAuthentication yes
+          AuthenticationMethods publickey
+          UsePAM yes
+
+          {% if not proxmox_check.stat.exists %}
+          PermitRootLogin no
+          {% else %}
+          PermitRootLogin prohibit-password
+          {% endif %}
+        validate: 'sshd -t -f %s'
+      notify: Restart SSH
+
+  handlers:
+    - name: Restart SSH
+      ansible.builtin.service:
+        name: "{{ 'sshd'
+                  if ansible_facts.os_family in
+                  ['RedHat','Rocky','AlmaLinux','Fedora','OracleLinux','Suse']
+                  else 'ssh' }}"
+        state: restarted
+
+# ==============================
+# THIRD PLAY: HOSTNAME
+# ==============================
+
+- name: Set hostname from inventory
+  hosts: all
+  become: true
+  tags: never,hostname
+
+  tasks:
+
+    - name: Set system hostname to inventory_hostname
+      ansible.builtin.hostname:
+        name: "{{ inventory_hostname }}"
+
+# ==============================
+# FOURTH PLAY: FREEIPA / SSSD
+# ==============================
+
+- name: FreeIPA client setup
+  hosts: all
+  become: true
+  tags: never,sssd
+
+  roles:
+    - role: initial_install/roles/freeipa_client
+
+# ==============================
+# FIFTH PLAY: BACKUP
+# ==============================
+
+- import_playbook: backup.yml

test_sms.yml

@@ -0,0 +1,57 @@
+---
+- name: Send and verify SMS delivery via internet-master.cz
+  hosts: localhost
+  gather_facts: false
+  vars:
+    sms_number: "601358865"
+    sms_username: "mikrotik"
+    sms_password_send: "jdkotzHJIOPWhjtr32D"
+    sms_password_recv: "jdkotzHJIOPWhjtr32D"
+    sms_wait_seconds: 15   # Wait 15s for delivery
+
+  tasks:
+    - name: Generate random test string
+      set_fact:
+        random_string: "mikrotik_{{ lookup('password', '/dev/null length=8 chars=ascii_letters') }}"
+
+    - name: Send SMS message
+      uri:
+        url: "https://sms.internet-master.cz/send/?number={{ sms_number }}&message=@mikrotik@{{ random_string | urlencode }}&type=class-1&username={{ sms_username }}&password={{ sms_password_send }}"
+        method: GET
+        return_content: true
+      register: send_result
+
+    - name: Show send API response
+      debug:
+        var: send_result.content
+
+    - name: Wait for SMS to be delivered
+      pause:
+        seconds: "{{ sms_wait_seconds }}"
+
+    - name: Fetch received messages
+      uri:
+        url: "https://sms.internet-master.cz/receive/?username={{ sms_username }}&password={{ sms_password_recv }}"
+        method: GET
+        return_content: true
+      register: recv_result
+
+    - name: Parse received JSON
+      set_fact:
+        inbox: "{{ recv_result.json.inbox | default([]) }}"
+
+    - name: Check if random string message was received
+      set_fact:
+        message_found: "{{ inbox | selectattr('message', 'equalto', random_string) | list | length > 0 }}"
+
+    - name: Report result
+      debug:
+        msg: >
+          SMS with message '{{ random_string }}' was {{
+            'delivered ✅' if message_found else 'NOT delivered ❌'
+          }}.
+
+    - name: Fail if not delivered
+      fail:
+        msg: "Message '{{ random_string }}' not found in received inbox!"
+      when: not message_found

update.yml

@@ -1,34 +1,91 @@
+---
 - name: Update system (APT + Flatpak)
   hosts: all
+  become: yes
+  gather_facts: yes
+  serial: 5
 
-  become: true
-  become_user: root
-  become_method: sudo
   tasks:
-    - name: Update APT cache
+
+    - name: Ensure SSH is reachable (skip host if not)
+      delegate_to: localhost
+      wait_for:
+        host: "{{ ansible_host | default(inventory_hostname) }}"
+        port: 22
+        timeout: 5
+      register: ssh_check
+      ignore_errors: yes
+
+    - meta: end_host
+      when: ssh_check is failed
+
+    - name: Ping with retries (handle intermittent flaps)
+      ping:
+      register: ping_result
+      retries: 5
+      delay: 5
+      until: ping_result is success
+
+    - name: Wait for apt lock to be released
+      shell: |
+        while fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1; do
+          echo "Waiting for apt lock..."
+          sleep 5
+        done
+      changed_when: false
+
+    - name: Update apt cache
       apt:
         update_cache: yes
 
-    - name: Upgrade all APT packages
+    - name: Perform full upgrade
       apt:
-        upgrade: dist
+        upgrade: full
+        autoremove: yes
+        autoclean: yes
+      register: apt_upgrade
+      retries: 3
+      delay: 10
+      until: apt_upgrade is succeeded
 
-    - name: Check if flatpak binary exists
-      stat:
-        path: /usr/bin/flatpak
-      register: flatpak_bin
+    - name: Fix broken packages
+      command: apt-get -f install -y
+      register: fix_result
+      failed_when: false
+      changed_when: "'Setting up' in fix_result.stdout"
+
+    - name: Check if Flatpak is installed
+      command: which flatpak
+      register: flatpak_check
+      changed_when: false
+      failed_when: false
 
     - name: Update system Flatpaks
-      shell: timeout 300 flatpak update -y
-      register: flatpak_sys
-      failed_when: flatpak_sys.rc != 0 and flatpak_sys.rc != 124
-      when: flatpak_bin.stat.exists
+      command: flatpak update -y --noninteractive --system
+      when: flatpak_check.rc == 0
+      failed_when: false
 
     - name: Update user Flatpaks
-      become_user: jakub
-      environment:
-        XDG_RUNTIME_DIR: /run/user/1000
-      shell: timeout 300 flatpak update -y
-      register: flatpak_user
-      failed_when: flatpak_user.rc != 0 and flatpak_user.rc != 124
-      when: flatpak_bin.stat.exists
+      command: flatpak update -y --noninteractive --user
+      become: false
+      when: flatpak_check.rc == 0
+      failed_when: false
+
+    - name: Remove unused Flatpaks
+      command: flatpak uninstall -y --noninteractive --unused
+      when: flatpak_check.rc == 0
+      failed_when: false
+    
+    - name: Update snap packages
+      command: snap refresh
+      failed_when: false
+    
+    - name: Check if reboot is required
+      stat:
+        path: /var/run/reboot-required
+      register: reboot_required
+
+    - name: Notify if reboot required
+      debug:
+        msg: "Reboot required on {{ inventory_hostname }}"
+      when: reboot_required.stat.exists

users-ssh.yml

@@ -1,15 +1,12 @@
-# users-ssh-nopasswd.yml
+---
 - name: Ensure users, SSH keys, and passwordless sudo
   hosts: all
   become: true
-  become_user: root
-  become_method: sudo
 
   vars:
     users:
       - name: automation
         shell: /bin/bash
-        # optional extra groups besides sudo/wheel
         groups: []
         sudo_nopasswd: true
         keys:
@@ -30,33 +27,43 @@
           - "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPFS4fsqMjMMu/Bi/884bw7yJBqvWusDRESvanH6Owco jakub@jimbuntu"
 
   tasks:
+
     - name: Pick sudo group per distro
       ansible.builtin.set_fact:
-        sudo_group: "{{ 'wheel' if ansible_facts.os_family in ['RedHat','Rocky','AlmaLinux','Fedora','OracleLinux','Suse'] else 'sudo' }}"
+        sudo_group: >-
+          {{ 'wheel'
+             if ansible_facts.os_family in
+             ['RedHat','Rocky','AlmaLinux','Fedora','OracleLinux','Suse']
+             else 'sudo' }}
 
     - name: Ensure user exists (creates home)
       ansible.builtin.user:
         name: "{{ item.name }}"
-        shell: "{{ item.shell | default('/bin/bash') }}"
+        shell: "{{ item.shell | default(omit) }}"
         groups: >-
           {{ (
-                (item.groups | default([]))
-                + ([sudo_group] if item.sudo_nopasswd | default(false) else [])
-              ) | unique | join(',') if
-              ((item.groups | default([])) | length > 0) or (item.sudo_nopasswd | default(false))
-              else omit }}
+               (item.groups | default([]))
+               + ([sudo_group] if item.sudo_nopasswd | default(false) else [])
+             ) | unique | join(',')
+             if (
+               (item.groups | default([]) | length > 0)
+               or item.sudo_nopasswd | default(false)
+             )
+             else omit }}
         append: true
         create_home: true
         state: present
       loop: "{{ users }}"
 
-    - name: Install authorized SSH keys
+    - name: Enforce authorized SSH keys
       ansible.builtin.authorized_key:
-        user: "{{ item.0.name }}"
-        key: "{{ item.1 }}"
+        user: "{{ item.name }}"
+        key: "{{ item.keys | join('\n') }}"
         state: present
         manage_dir: true
-      loop: "{{ users | subelements('keys', skip_missing=True) }}"
+        exclusive: true
+      loop: "{{ users }}"
+      when: item.keys is defined
 
     - name: Grant passwordless sudo via sudoers.d
       ansible.builtin.copy:
@@ -64,7 +71,9 @@
         owner: root
         group: root
         mode: '0440'
-        content: "{{ item.name }} ALL=(ALL) NOPASSWD:ALL"
+        content: |
+          # Managed by Ansible
+          {{ item.name }} ALL=(ALL) NOPASSWD:ALL
         validate: 'visudo -cf %s'
       when: item.sudo_nopasswd | default(false)
       loop: "{{ users }}"