ansible/update_semaphore.yml

# update_semaphore.yml

- name: Update Semaphore (self-update safe)
  hosts: pve2_vm
  gather_facts: false

  vars:
    # Compose sync (controller -> target)
    compose_local_dir: "{{ playbook_dir }}/docker-compose"
    compose_remote_base: "/home/{{ ansible_user }}/.ansible-compose"
    compose_remote_dir: "{{ compose_remote_base }}/docker-compose"
    compose_remote_archive: "{{ compose_remote_base }}/docker-compose.tar.gz"

    # Semaphore settings
    semaphore_project: semaphore
    semaphore_container: semaphore
    semaphore_compose_filename: "docker-compose-semaphore.yml"
    # Published port on the VM. NOT 3000 — that only exists in the container.
    semaphore_port: 3008

    semaphore_update_log: "{{ compose_remote_base }}/semaphore-update.log"

  tasks:
    # This job gets killed when the container is recreated, so the final
    # verification lives in the log and is reported on the next run.
    - name: Show result of the previous update
      ansible.builtin.command:
        argv: [tail, -n, "12", "{{ semaphore_update_log }}"]
      register: prev_log
      changed_when: false
      failed_when: false

    - name: Previous update result
      ansible.builtin.debug:
        var: prev_log.stdout_lines

    # Env vars override /etc/semaphore/config.json. A stray SEMAPHORE_DB_*
    # points Semaphore at a different database; a stray
    # SEMAPHORE_ACCESS_KEY_ENCRYPTION makes every credential undecryptable.
    - name: Refuse compose that overrides DB or encryption config
      ansible.builtin.assert:
        that:
          - forbidden | length == 0
        fail_msg: "Compose sets forbidden env vars: {{ forbidden }}. Remove them."
        success_msg: "Compose does not override DB or encryption config."
      vars:
        forbidden: >-
          {{ lookup('file', compose_local_dir ~ '/' ~ semaphore_compose_filename)
             | regex_findall('(?m)^\s*(SEMAPHORE_DB_\w+|SEMAPHORE_ACCESS_KEY_ENCRYPTION)\s*:') }}
      delegate_to: localhost
      run_once: true

    - name: Ensure remote base directory exists
      ansible.builtin.file:
        path: "{{ compose_remote_base }}"
        state: directory
        mode: "0755"

    - name: Create local archive of docker-compose directory (controller)
      ansible.builtin.archive:
        path: "{{ compose_local_dir }}/"
        dest: "/tmp/docker-compose.tar.gz"
        format: gz
      delegate_to: localhost
      run_once: true

    - name: Upload archive to remote host
      ansible.builtin.copy:
        src: "/tmp/docker-compose.tar.gz"
        dest: "{{ compose_remote_archive }}"
        mode: "0644"

    - name: Recreate remote compose directory
      ansible.builtin.file:
        path: "{{ compose_remote_dir }}"
        state: absent

    - name: Ensure remote compose directory exists
      ansible.builtin.file:
        path: "{{ compose_remote_dir }}"
        state: directory
        mode: "0755"

    - name: Extract archive on remote host
      ansible.builtin.unarchive:
        src: "{{ compose_remote_archive }}"
        dest: "{{ compose_remote_dir }}"
        remote_src: true

    # Online snapshot via the sqlite backup API — consistent without stopping
    # the server. Reads the DB path from the live config instead of guessing.
    - name: Back up Semaphore database
      ansible.builtin.command:
        argv:
          - docker
          - exec
          - "{{ semaphore_container }}"
          - python3
          - -c
          - |
            import json, os, sqlite3, datetime, glob
            cfg = json.load(open('/etc/semaphore/config.json'))
            out = '/etc/semaphore/backups'
            os.makedirs(out, exist_ok=True)
            ts = datetime.datetime.utcnow().strftime('%Y%m%d-%H%M%S')
            dst = f'{out}/database-{ts}.sqlite'
            a = sqlite3.connect('file:' + cfg['sqlite']['host'] + '?mode=ro', uri=True)
            b = sqlite3.connect(dst)
            with b:
                a.backup(b)
            b.close(); a.close()
            json.dump(cfg, open(f'{out}/config-{ts}.json', 'w'), indent=2)
            print(dst, os.path.getsize(dst), 'bytes')
            for old in sorted(glob.glob(f'{out}/database-*.sqlite'), reverse=True)[10:]:
                os.remove(old)
      register: sem_db_backup
      changed_when: true

    - name: Database backup result
      ansible.builtin.debug:
        var: sem_db_backup.stdout_lines

    - name: Pull latest Semaphore image
      community.docker.docker_compose_v2:
        project_name: "{{ semaphore_project }}"
        project_src: "{{ compose_remote_dir }}"
        files:
          - "{{ semaphore_compose_filename }}"
        pull: always

    # Cannot use docker_compose_v2 here: recreating the container kills this
    # very Ansible process. Detached so the update outlives the job.
    # No `recreate: always` — compose recreates only when the image or the
    # resolved config actually changed, so unchanged runs finish green.
    - name: Recreate Semaphore in the background
      ansible.builtin.shell: |
        cat > "{{ compose_remote_base }}/semaphore-selfupdate.sh" <<'SH'
        #!/usr/bin/env bash
        set -uo pipefail
        exec 2>&1
        cd "{{ compose_remote_dir }}" || exit 1
        echo "=== $(date -Is) update starting ==="
        docker compose -p "{{ semaphore_project }}" \
          -f "{{ semaphore_compose_filename }}" \
          up -d --remove-orphans --wait --wait-timeout 300
        for _ in $(seq 1 60); do
          code=$(curl -s -o /dev/null -w '%{http_code}' \
            "http://127.0.0.1:{{ semaphore_port }}/api/ping" || true)
          if [ "$code" = "200" ]; then
            docker logs --tail 40 "{{ semaphore_container }}" 2>&1 \
              | grep -m1 'SQLite @' || echo "WARN: no 'SQLite @' line"
            echo "OK: semaphore healthy at $(date -Is)"
            exit 0
          fi
          sleep 5
        done
        echo "FAIL: no 200 from /api/ping after 300s" >&2
        exit 1
        SH
        chmod +x "{{ compose_remote_base }}/semaphore-selfupdate.sh"
        setsid nohup "{{ compose_remote_base }}/semaphore-selfupdate.sh" \
          > "{{ semaphore_update_log }}" 2>&1 < /dev/null &
        echo launched
      args:
        executable: /bin/bash
      changed_when: true

    - name: What happens next
      ansible.builtin.debug:
        msg:
          - "Update launched detached. If the image changed, this job dies here."
          - "That is expected — the log is the success signal, not the task status."
          - "tail -n 20 {{ semaphore_update_log }}"