fix: update semaphore configuration and enhance update process in Ansible playbook

docker-compose/docker-compose-semaphore.yml

@@ -2,21 +2,17 @@
 
 services:
   semaphore:
-    # Official upstream image, same version as the custom build.
-    # Changing image and version separately keeps failures diagnosable.
     image: semaphoreui/semaphore:v2.18.3
     container_name: semaphore
     user: "0:0"
     ports:
       - "3008:3000"
     environment:
-      # Database config lives in /etc/semaphore/config.json
       SEMAPHORE_TMP_PATH: /var/lib/semaphore/projects
       SEMAPHORE_ADMIN: admin
       SEMAPHORE_ADMIN_NAME: admin
       SEMAPHORE_ADMIN_EMAIL: admin@localhost
     env_file:
-      # Absolute path — the deploy dir is wiped on every playbook run
       - /data/compose/semaphore/semaphore.env
     volumes:
       - /data/compose/semaphore/db:/etc/semaphore

update_semaphore.yml

@@ -5,20 +5,56 @@
   gather_facts: false
 
   vars:
+    # Compose sync (controller -> target)
     compose_local_dir: "{{ playbook_dir }}/docker-compose"
     compose_remote_base: "/home/{{ ansible_user }}/.ansible-compose"
     compose_remote_dir: "{{ compose_remote_base }}/docker-compose"
     compose_remote_archive: "{{ compose_remote_base }}/docker-compose.tar.gz"
 
+    # Semaphore settings
     semaphore_project: semaphore
+    semaphore_container: semaphore
     semaphore_compose_filename: "docker-compose-semaphore.yml"
-    semaphore_port: 3000
+    # Published port on the VM. NOT 3000 — that only exists in the container.
+    semaphore_port: 3008
+
+    semaphore_update_log: "{{ compose_remote_base }}/semaphore-update.log"
 
   tasks:
+    # This job gets killed when the container is recreated, so the final
+    # verification lives in the log and is reported on the next run.
+    - name: Show result of the previous update
+      ansible.builtin.command:
+        argv: [tail, -n, "12", "{{ semaphore_update_log }}"]
+      register: prev_log
+      changed_when: false
+      failed_when: false
+
+    - name: Previous update result
+      ansible.builtin.debug:
+        var: prev_log.stdout_lines
+
+    # Env vars override /etc/semaphore/config.json. A stray SEMAPHORE_DB_*
+    # points Semaphore at a different database; a stray
+    # SEMAPHORE_ACCESS_KEY_ENCRYPTION makes every credential undecryptable.
+    - name: Refuse compose that overrides DB or encryption config
+      ansible.builtin.assert:
+        that:
+          - forbidden | length == 0
+        fail_msg: "Compose sets forbidden env vars: {{ forbidden }}. Remove them."
+        success_msg: "Compose does not override DB or encryption config."
+      vars:
+        forbidden: >-
+          {{ lookup('file', compose_local_dir ~ '/' ~ semaphore_compose_filename)
+             | regex_findall('(?m)^\s*(SEMAPHORE_DB_\w+|SEMAPHORE_ACCESS_KEY_ENCRYPTION)\s*:') }}
+      delegate_to: localhost
+      run_once: true
+
     - name: Ensure remote base directory exists
       ansible.builtin.file:
         path: "{{ compose_remote_base }}"
         state: directory
+        mode: "0755"
 
     - name: Create local archive of docker-compose directory (controller)
       ansible.builtin.archive:
@@ -32,6 +68,7 @@
       ansible.builtin.copy:
         src: "/tmp/docker-compose.tar.gz"
         dest: "{{ compose_remote_archive }}"
+        mode: "0644"
 
     - name: Recreate remote compose directory
       ansible.builtin.file:
@@ -42,6 +79,7 @@
       ansible.builtin.file:
         path: "{{ compose_remote_dir }}"
         state: directory
+        mode: "0755"
 
     - name: Extract archive on remote host
       ansible.builtin.unarchive:
@@ -49,45 +87,87 @@
         dest: "{{ compose_remote_dir }}"
         remote_src: true
 
-    - name: Pull latest Semaphore image(s)
+    # Online snapshot via the sqlite backup API — consistent without stopping
+    # the server. Reads the DB path from the live config instead of guessing.
+    - name: Back up Semaphore database
       ansible.builtin.command:
         argv:
-          - bash
+          - docker
-          - -lc
+          - exec
-          - >
+          - "{{ semaphore_container }}"
-            cd "{{ compose_remote_dir }}"
+          - python3
-            && docker compose -p "{{ semaphore_project }}"
+          - -c
-            -f "{{ semaphore_compose_filename }}"
+          - |
-            pull
+            import json, os, sqlite3, datetime, glob
+            cfg = json.load(open('/etc/semaphore/config.json'))
+            out = '/etc/semaphore/backups'
+            os.makedirs(out, exist_ok=True)
+            ts = datetime.datetime.utcnow().strftime('%Y%m%d-%H%M%S')
+            dst = f'{out}/database-{ts}.sqlite'
+            a = sqlite3.connect('file:' + cfg['sqlite']['host'] + '?mode=ro', uri=True)
+            b = sqlite3.connect(dst)
+            with b:
+                a.backup(b)
+            b.close(); a.close()
+            json.dump(cfg, open(f'{out}/config-{ts}.json', 'w'), indent=2)
+            print(dst, os.path.getsize(dst), 'bytes')
+            for old in sorted(glob.glob(f'{out}/database-*.sqlite'), reverse=True)[10:]:
+                os.remove(old)
+      register: sem_db_backup
       changed_when: true
 
-    - name: Start Semaphore update in background (avoid killing this job)
+    - name: Database backup result
-      ansible.builtin.command:
+      ansible.builtin.debug:
-        argv:
+        var: sem_db_backup.stdout_lines
-          - bash
+
-          - -lc
+    - name: Pull latest Semaphore image
-          - >
+      community.docker.docker_compose_v2:
-            cd "{{ compose_remote_dir }}"
+        project_name: "{{ semaphore_project }}"
-            && nohup docker compose -p "{{ semaphore_project }}"
+        project_src: "{{ compose_remote_dir }}"
-            -f "{{ semaphore_compose_filename }}"
+        files:
-            up -d --remove-orphans --force-recreate
+          - "{{ semaphore_compose_filename }}"
-            > "{{ compose_remote_base }}/semaphore-update.log" 2>&1 &
+        pull: always
-      async: 1
+
-      poll: 0
+    # Cannot use docker_compose_v2 here: recreating the container kills this
+    # very Ansible process. Detached so the update outlives the job.
+    # No `recreate: always` — compose recreates only when the image or the
+    # resolved config actually changed, so unchanged runs finish green.
+    - name: Recreate Semaphore in the background
+      ansible.builtin.shell: |
+        cat > "{{ compose_remote_base }}/semaphore-selfupdate.sh" <<'SH'
+        #!/usr/bin/env bash
+        set -uo pipefail
+        exec 2>&1
+        cd "{{ compose_remote_dir }}" || exit 1
+        echo "=== $(date -Is) update starting ==="
+        docker compose -p "{{ semaphore_project }}" \
+          -f "{{ semaphore_compose_filename }}" \
+          up -d --remove-orphans --wait --wait-timeout 300
+        for _ in $(seq 1 60); do
+          code=$(curl -s -o /dev/null -w '%{http_code}' \
+            "http://127.0.0.1:{{ semaphore_port }}/api/ping" || true)
+          if [ "$code" = "200" ]; then
+            docker logs --tail 40 "{{ semaphore_container }}" 2>&1 \
+              | grep -m1 'SQLite @' || echo "WARN: no 'SQLite @' line"
+            echo "OK: semaphore healthy at $(date -Is)"
+            exit 0
+          fi
+          sleep 5
+        done
+        echo "FAIL: no 200 from /api/ping after 300s" >&2
+        exit 1
+        SH
+        chmod +x "{{ compose_remote_base }}/semaphore-selfupdate.sh"
+        setsid nohup "{{ compose_remote_base }}/semaphore-selfupdate.sh" \
+          > "{{ semaphore_update_log }}" 2>&1 < /dev/null &
+        echo launched
+      args:
+        executable: /bin/bash
       changed_when: true
 
-    - name: Wait for Semaphore port
+    - name: What happens next
-      ansible.builtin.wait_for:
+      ansible.builtin.debug:
-        host: 127.0.0.1
+        msg:
-        port: "{{ semaphore_port }}"
+          - "Update launched detached. If the image changed, this job dies here."
-        timeout: 300
+          - "That is expected — the log is the success signal, not the task status."
-
+          - "tail -n 20 {{ semaphore_update_log }}"
-    - name: Check Semaphore HTTP endpoint (retry)
-      ansible.builtin.uri:
-        url: "http://127.0.0.1:{{ semaphore_port }}/"
-        status_code: 200
-      register: sem_http
-      retries: 30
-      delay: 5
-      until: sem_http.status == 200
-      changed_when: false