ansible/update_semaphore.yml

# update_semaphore.yml

- name: Update Semaphore (self-update safe)
  hosts: pve2_vm
  gather_facts: false

  vars:
    # Compose sync (controller -> target)
    compose_local_dir: "{{ playbook_dir }}/docker-compose"
    compose_remote_base: "/home/{{ ansible_user }}/.ansible-compose"
    compose_remote_dir: "{{ compose_remote_base }}/docker-compose"
    compose_remote_archive: "{{ compose_remote_base }}/docker-compose.tar.gz"

    # Semaphore settings
    semaphore_project: semaphore
    semaphore_container: semaphore
    semaphore_compose_filename: "docker-compose-semaphore.yml"
    semaphore_port: 3008

    semaphore_update_log: "{{ compose_remote_base }}/semaphore-update.log"
    semaphore_update_delay: 20

  tasks:
    - name: Show result of the previous update
      ansible.builtin.command:
        argv: [tail, -n, "12", "{{ semaphore_update_log }}"]
      register: prev_log
      changed_when: false
      failed_when: false

    - name: Previous update result
      ansible.builtin.debug:
        var: prev_log.stdout_lines

    - name: Refuse compose that overrides DB or encryption config
      ansible.builtin.assert:
        that:
          - forbidden | length == 0
        fail_msg: "Compose sets forbidden env vars: {{ forbidden }}. Remove them."
        success_msg: "Compose does not override DB or encryption config."
      vars:
        forbidden: >-
          {{ lookup('file', compose_local_dir ~ '/' ~ semaphore_compose_filename)
             | regex_findall('(?m)^\s*(SEMAPHORE_DB_\w+|SEMAPHORE_ACCESS_KEY_ENCRYPTION)\s*:') }}
      delegate_to: localhost
      run_once: true

    - name: Ensure remote base directory exists
      ansible.builtin.file:
        path: "{{ compose_remote_base }}"
        state: directory
        mode: "0755"

    - name: Create local archive of docker-compose directory (controller)
      ansible.builtin.archive:
        path: "{{ compose_local_dir }}/"
        dest: "/tmp/docker-compose.tar.gz"
        format: gz
      delegate_to: localhost
      run_once: true

    - name: Upload archive to remote host
      ansible.builtin.copy:
        src: "/tmp/docker-compose.tar.gz"
        dest: "{{ compose_remote_archive }}"
        mode: "0644"

    - name: Recreate remote compose directory
      ansible.builtin.file:
        path: "{{ compose_remote_dir }}"
        state: absent

    - name: Ensure remote compose directory exists
      ansible.builtin.file:
        path: "{{ compose_remote_dir }}"
        state: directory
        mode: "0755"

    - name: Extract archive on remote host
      ansible.builtin.unarchive:
        src: "{{ compose_remote_archive }}"
        dest: "{{ compose_remote_dir }}"
        remote_src: true

    - name: Back up Semaphore database
      ansible.builtin.command:
        argv:
          - docker
          - exec
          - "{{ semaphore_container }}"
          - python3
          - -c
          - |
            import json, os, sqlite3, datetime, glob
            cfg = json.load(open('/etc/semaphore/config.json'))
            out = '/etc/semaphore/backups'
            os.makedirs(out, exist_ok=True)
            ts = datetime.datetime.now(datetime.timezone.utc).strftime('%Y%m%d-%H%M%S')
            dst = f'{out}/database-{ts}.sqlite'
            a = sqlite3.connect('file:' + cfg['sqlite']['host'] + '?mode=ro', uri=True)
            b = sqlite3.connect(dst)
            with b:
                a.backup(b)
            b.close(); a.close()
            json.dump(cfg, open(f'{out}/config-{ts}.json', 'w'), indent=2)
            print(dst, os.path.getsize(dst), 'bytes')
            for old in sorted(glob.glob(f'{out}/database-*.sqlite'), reverse=True)[10:]:
                os.remove(old)
                stale = f"{out}/config-{os.path.basename(old)[9:-7]}.json"
                if os.path.exists(stale):
                    os.remove(stale)
                print('pruned', old)
      register: sem_db_backup
      changed_when: true

    - name: Database backup result
      ansible.builtin.debug:
        var: sem_db_backup.stdout_lines

    - name: Recreate Semaphore in the background
      ansible.builtin.shell: |
        cat > "{{ compose_remote_base }}/semaphore-selfupdate.sh" <<'SH'
        #!/usr/bin/env bash
        set -uo pipefail
        exec 2>&1
        cd "{{ compose_remote_dir }}" || exit 1
        COMPOSE=(docker compose -p "{{ semaphore_project }}" -f "{{ semaphore_compose_filename }}")

        # Let the calling job finish and its status get written to the DB
        # before we potentially replace the container running it.
        sleep {{ semaphore_update_delay }}

        echo "=== $(date -Is) update starting ==="
        "${COMPOSE[@]}" pull || { echo "FAIL: pull failed" >&2; exit 1; }

        # No --force-recreate: compose replaces the container only when the
        # image or the resolved config (including env_file) actually changed,
        # so unchanged runs cause no downtime.
        "${COMPOSE[@]}" up -d --remove-orphans --wait --wait-timeout 300

        for _ in $(seq 1 60); do
          code=$(curl -s -o /dev/null -w '%{http_code}' \
            "http://127.0.0.1:{{ semaphore_port }}/api/ping" || true)
          if [ "$code" = "200" ]; then
            echo "image: $(docker inspect "{{ semaphore_container }}" \
              --format '{{ '{{' }}.Config.Image{{ '}}' }}')"
            # Grep the whole log: when the container was not replaced, the
            # startup line is far outside the last few lines.
            docker logs "{{ semaphore_container }}" 2>&1 \
              | grep -m1 'SQLite @' || echo "WARN: no 'SQLite @' line"
            echo "OK: semaphore healthy at $(date -Is)"
            exit 0
          fi
          sleep 5
        done
        echo "FAIL: no 200 from /api/ping after 300s" >&2
        exit 1
        SH
        chmod +x "{{ compose_remote_base }}/semaphore-selfupdate.sh"
        setsid nohup "{{ compose_remote_base }}/semaphore-selfupdate.sh" \
          > "{{ semaphore_update_log }}" 2>&1 < /dev/null &
        echo launched
      args:
        executable: /bin/bash
      changed_when: true

    - name: What happens next
      ansible.builtin.debug:
        msg:
          - "Update starts in {{ semaphore_update_delay }}s, after this job ends."
          - "If the image changed, the container is replaced — expected."
          - "The log is the success signal: tail -n 20 {{ semaphore_update_log }}"