From 17c4a696d25a01f4c93989cf8c75fea1869f0b4e Mon Sep 17 00:00:00 2001 From: "martin.fencl" Date: Mon, 17 Aug 2026 23:19:58 +0200 Subject: [PATCH] fix: update semaphore configuration and enhance update process in Ansible playbook --- docker-compose/docker-compose-semaphore.yml | 4 - update_semaphore.yml | 154 +++++++++++++++----- 2 files changed, 117 insertions(+), 41 deletions(-) diff --git a/docker-compose/docker-compose-semaphore.yml b/docker-compose/docker-compose-semaphore.yml index 7f5ee8b..f04a635 100644 --- a/docker-compose/docker-compose-semaphore.yml +++ b/docker-compose/docker-compose-semaphore.yml @@ -2,21 +2,17 @@ services: semaphore: - # Official upstream image, same version as the custom build. - # Changing image and version separately keeps failures diagnosable. image: semaphoreui/semaphore:v2.18.3 container_name: semaphore user: "0:0" ports: - "3008:3000" environment: - # Database config lives in /etc/semaphore/config.json SEMAPHORE_TMP_PATH: /var/lib/semaphore/projects SEMAPHORE_ADMIN: admin SEMAPHORE_ADMIN_NAME: admin SEMAPHORE_ADMIN_EMAIL: admin@localhost env_file: - # Absolute path — the deploy dir is wiped on every playbook run - /data/compose/semaphore/semaphore.env volumes: - /data/compose/semaphore/db:/etc/semaphore diff --git a/update_semaphore.yml b/update_semaphore.yml index 76e69e7..8123916 100644 --- a/update_semaphore.yml +++ b/update_semaphore.yml @@ -5,20 +5,56 @@ gather_facts: false vars: + # Compose sync (controller -> target) compose_local_dir: "{{ playbook_dir }}/docker-compose" compose_remote_base: "/home/{{ ansible_user }}/.ansible-compose" compose_remote_dir: "{{ compose_remote_base }}/docker-compose" compose_remote_archive: "{{ compose_remote_base }}/docker-compose.tar.gz" + # Semaphore settings semaphore_project: semaphore + semaphore_container: semaphore semaphore_compose_filename: "docker-compose-semaphore.yml" - semaphore_port: 3000 + # Published port on the VM. NOT 3000 — that only exists in the container. + semaphore_port: 3008 + + semaphore_update_log: "{{ compose_remote_base }}/semaphore-update.log" tasks: + # This job gets killed when the container is recreated, so the final + # verification lives in the log and is reported on the next run. + - name: Show result of the previous update + ansible.builtin.command: + argv: [tail, -n, "12", "{{ semaphore_update_log }}"] + register: prev_log + changed_when: false + failed_when: false + + - name: Previous update result + ansible.builtin.debug: + var: prev_log.stdout_lines + + # Env vars override /etc/semaphore/config.json. A stray SEMAPHORE_DB_* + # points Semaphore at a different database; a stray + # SEMAPHORE_ACCESS_KEY_ENCRYPTION makes every credential undecryptable. + - name: Refuse compose that overrides DB or encryption config + ansible.builtin.assert: + that: + - forbidden | length == 0 + fail_msg: "Compose sets forbidden env vars: {{ forbidden }}. Remove them." + success_msg: "Compose does not override DB or encryption config." + vars: + forbidden: >- + {{ lookup('file', compose_local_dir ~ '/' ~ semaphore_compose_filename) + | regex_findall('(?m)^\s*(SEMAPHORE_DB_\w+|SEMAPHORE_ACCESS_KEY_ENCRYPTION)\s*:') }} + delegate_to: localhost + run_once: true + - name: Ensure remote base directory exists ansible.builtin.file: path: "{{ compose_remote_base }}" state: directory + mode: "0755" - name: Create local archive of docker-compose directory (controller) ansible.builtin.archive: @@ -32,6 +68,7 @@ ansible.builtin.copy: src: "/tmp/docker-compose.tar.gz" dest: "{{ compose_remote_archive }}" + mode: "0644" - name: Recreate remote compose directory ansible.builtin.file: @@ -42,6 +79,7 @@ ansible.builtin.file: path: "{{ compose_remote_dir }}" state: directory + mode: "0755" - name: Extract archive on remote host ansible.builtin.unarchive: @@ -49,45 +87,87 @@ dest: "{{ compose_remote_dir }}" remote_src: true - - name: Pull latest Semaphore image(s) + # Online snapshot via the sqlite backup API — consistent without stopping + # the server. Reads the DB path from the live config instead of guessing. + - name: Back up Semaphore database ansible.builtin.command: argv: - - bash - - -lc - - > - cd "{{ compose_remote_dir }}" - && docker compose -p "{{ semaphore_project }}" - -f "{{ semaphore_compose_filename }}" - pull + - docker + - exec + - "{{ semaphore_container }}" + - python3 + - -c + - | + import json, os, sqlite3, datetime, glob + cfg = json.load(open('/etc/semaphore/config.json')) + out = '/etc/semaphore/backups' + os.makedirs(out, exist_ok=True) + ts = datetime.datetime.utcnow().strftime('%Y%m%d-%H%M%S') + dst = f'{out}/database-{ts}.sqlite' + a = sqlite3.connect('file:' + cfg['sqlite']['host'] + '?mode=ro', uri=True) + b = sqlite3.connect(dst) + with b: + a.backup(b) + b.close(); a.close() + json.dump(cfg, open(f'{out}/config-{ts}.json', 'w'), indent=2) + print(dst, os.path.getsize(dst), 'bytes') + for old in sorted(glob.glob(f'{out}/database-*.sqlite'), reverse=True)[10:]: + os.remove(old) + register: sem_db_backup changed_when: true - - name: Start Semaphore update in background (avoid killing this job) - ansible.builtin.command: - argv: - - bash - - -lc - - > - cd "{{ compose_remote_dir }}" - && nohup docker compose -p "{{ semaphore_project }}" - -f "{{ semaphore_compose_filename }}" - up -d --remove-orphans --force-recreate - > "{{ compose_remote_base }}/semaphore-update.log" 2>&1 & - async: 1 - poll: 0 + - name: Database backup result + ansible.builtin.debug: + var: sem_db_backup.stdout_lines + + - name: Pull latest Semaphore image + community.docker.docker_compose_v2: + project_name: "{{ semaphore_project }}" + project_src: "{{ compose_remote_dir }}" + files: + - "{{ semaphore_compose_filename }}" + pull: always + + # Cannot use docker_compose_v2 here: recreating the container kills this + # very Ansible process. Detached so the update outlives the job. + # No `recreate: always` — compose recreates only when the image or the + # resolved config actually changed, so unchanged runs finish green. + - name: Recreate Semaphore in the background + ansible.builtin.shell: | + cat > "{{ compose_remote_base }}/semaphore-selfupdate.sh" <<'SH' + #!/usr/bin/env bash + set -uo pipefail + exec 2>&1 + cd "{{ compose_remote_dir }}" || exit 1 + echo "=== $(date -Is) update starting ===" + docker compose -p "{{ semaphore_project }}" \ + -f "{{ semaphore_compose_filename }}" \ + up -d --remove-orphans --wait --wait-timeout 300 + for _ in $(seq 1 60); do + code=$(curl -s -o /dev/null -w '%{http_code}' \ + "http://127.0.0.1:{{ semaphore_port }}/api/ping" || true) + if [ "$code" = "200" ]; then + docker logs --tail 40 "{{ semaphore_container }}" 2>&1 \ + | grep -m1 'SQLite @' || echo "WARN: no 'SQLite @' line" + echo "OK: semaphore healthy at $(date -Is)" + exit 0 + fi + sleep 5 + done + echo "FAIL: no 200 from /api/ping after 300s" >&2 + exit 1 + SH + chmod +x "{{ compose_remote_base }}/semaphore-selfupdate.sh" + setsid nohup "{{ compose_remote_base }}/semaphore-selfupdate.sh" \ + > "{{ semaphore_update_log }}" 2>&1 < /dev/null & + echo launched + args: + executable: /bin/bash changed_when: true - - name: Wait for Semaphore port - ansible.builtin.wait_for: - host: 127.0.0.1 - port: "{{ semaphore_port }}" - timeout: 300 - - - name: Check Semaphore HTTP endpoint (retry) - ansible.builtin.uri: - url: "http://127.0.0.1:{{ semaphore_port }}/" - status_code: 200 - register: sem_http - retries: 30 - delay: 5 - until: sem_http.status == 200 - changed_when: false + - name: What happens next + ansible.builtin.debug: + msg: + - "Update launched detached. If the image changed, this job dies here." + - "That is expected — the log is the success signal, not the task status." + - "tail -n 20 {{ semaphore_update_log }}"