From 0d3363fcdecef7ade57b100fd44a466e7e1593fa Mon Sep 17 00:00:00 2001 From: "martin.fencl" Date: Thu, 6 Aug 2026 18:23:31 +0200 Subject: [PATCH] fix: enhance Nextcloud update playbook with improved backup and upgrade logic, and add container readiness checks --- nextcloud/update_nextcloud_v2.yml | 532 ++++++++++++++++++------------ 1 file changed, 315 insertions(+), 217 deletions(-) diff --git a/nextcloud/update_nextcloud_v2.yml b/nextcloud/update_nextcloud_v2.yml index 13e9629..7b92a05 100644 --- a/nextcloud/update_nextcloud_v2.yml +++ b/nextcloud/update_nextcloud_v2.yml @@ -8,160 +8,165 @@ become_method: sudo vars: - # Normalized values from Semaphore Variable Group + # vm_pass is stored in Semaphore Secrets as an environment variable. + vm_password: "{{ lookup('ansible.builtin.env', 'vm_pass') }}" + + # Values from Semaphore Extra variables. vm_use_sudo: "{{ use_sudo | default(false) | bool }}" debug_level: "{{ DEBUG | default(0) | int }}" retry_count: "{{ RETRIES | default(25) | int }}" - # Nextcloud configuration + # Nextcloud configuration. nextcloud_project: "nextcloud-collabora" nextcloud_compose_file: "/data/compose/nextcloud/docker-compose-nextcloud.yml" nextcloud_service: "nextcloud" nextcloud_container: "nextcloud" nextcloud_db_container: "nextcloud-db" + redis_container: "redis" nextcloud_base_url: "https://cloud.martinfencl.eu" nextcloud_status_url: "{{ nextcloud_base_url }}/status.php" + collabora_url: "https://collabora.martinfencl.eu/" + collabora_discovery_url: "https://collabora.martinfencl.eu/hosting/discovery" + backup_timestamp: "{{ lookup('pipe', 'date +%F-%H%M%S') }}" backup_dir: "/data/compose/nextcloud/backup-{{ backup_timestamp }}" - docker_prefix: >- - unalias docker 2>/dev/null || true; - export DOCKER_CLI_HINTS=0; - command docker + docker_cmd: "env DOCKER_CLI_HINTS=0 docker" nextcloud_version_command: >- - docker exec -u www-data - {{ nextcloud_container }} + {{ docker_cmd }} exec -u www-data + {{ nextcloud_container | quote }} php occ -V nextcloud_maintenance_off_command: >- - docker exec -u www-data - {{ nextcloud_container }} + {{ docker_cmd }} exec -u www-data + {{ nextcloud_container | quote }} php occ maintenance:mode --off - # Commands executed before the container update - nextcloud_backup_commands: - - >- - mkdir -p {{ backup_dir | quote }} + preflight_script: | + set -euo pipefail - - >- - {{ docker_prefix }} - exec -u www-data - {{ nextcloud_container }} + command -v docker >/dev/null + docker version >/dev/null + docker compose version >/dev/null + + test -f {{ nextcloud_compose_file | quote }} + + for container in \ + {{ nextcloud_container | quote }} \ + {{ nextcloud_db_container | quote }} \ + {{ redis_container | quote }} + do + state="$(docker inspect --format '{% raw %}{{.State.Running}}{% endraw %}' "${container}" 2>/dev/null || true)" + + if [ "${state}" != "true" ]; then + echo "Container ${container} is not running" >&2 + exit 1 + fi + done + + {{ docker_cmd }} exec -u www-data \ + {{ nextcloud_container | quote }} \ + php occ status + + backup_script: | + set -euo pipefail + + mkdir -p {{ backup_dir | quote }} + + echo "Enabling Nextcloud maintenance mode" + {{ docker_cmd }} exec -u www-data \ + {{ nextcloud_container | quote }} \ php occ maintenance:mode --on - - >- - {{ docker_prefix }} - exec - {{ nextcloud_container }} - sh -c - 'tar czf /tmp/nextcloud_conf.tgz - -C /var/www/html - config custom_apps' + echo "Backing up Nextcloud configuration and custom applications" + {{ docker_cmd }} exec \ + {{ nextcloud_container | quote }} \ + sh -c 'tar czf /tmp/nextcloud_conf.tgz -C /var/www/html config custom_apps' - - >- - {{ docker_prefix }} - cp - {{ nextcloud_container }}:/tmp/nextcloud_conf.tgz + {{ docker_cmd }} cp \ + {{ nextcloud_container | quote }}:/tmp/nextcloud_conf.tgz \ {{ (backup_dir ~ '/nextcloud_conf.tgz') | quote }} - - >- - {{ docker_prefix }} - exec - {{ nextcloud_container }} + {{ docker_cmd }} exec \ + {{ nextcloud_container | quote }} \ rm -f /tmp/nextcloud_conf.tgz - - >- - set -o pipefail; - {{ docker_prefix }} - exec - {{ nextcloud_db_container }} - sh -c - 'if command -v mariadb-dump >/dev/null 2>&1; then - exec mariadb-dump - -u"$MYSQL_USER" - -p"$MYSQL_PASSWORD" - "$MYSQL_DATABASE"; - else - exec mysqldump - -u"$MYSQL_USER" - -p"$MYSQL_PASSWORD" - "$MYSQL_DATABASE"; - fi' - > {{ (backup_dir ~ '/db.sql') | quote }} + echo "Backing up Nextcloud database" + {{ docker_cmd }} exec \ + {{ nextcloud_db_container | quote }} \ + sh -c ' + if command -v mariadb-dump >/dev/null 2>&1; then + exec mariadb-dump \ + -u"$MYSQL_USER" \ + -p"$MYSQL_PASSWORD" \ + "$MYSQL_DATABASE" + else + exec mysqldump \ + -u"$MYSQL_USER" \ + -p"$MYSQL_PASSWORD" \ + "$MYSQL_DATABASE" + fi + ' > {{ (backup_dir ~ '/db.sql') | quote }} - - >- - test -s - {{ (backup_dir ~ '/nextcloud_conf.tgz') | quote }} + test -s {{ (backup_dir ~ '/nextcloud_conf.tgz') | quote }} + test -s {{ (backup_dir ~ '/db.sql') | quote }} - - >- - test -s - {{ (backup_dir ~ '/db.sql') | quote }} + echo "Backup completed successfully: {{ backup_dir }}" - # Commands executed while maintenance mode is enabled - nextcloud_upgrade_commands: - - >- - {{ docker_prefix }} - compose - -p {{ nextcloud_project | quote }} - -f {{ nextcloud_compose_file | quote }} - pull + upgrade_script: | + set -euo pipefail + + echo "Pulling the current Nextcloud image" + {{ docker_cmd }} compose \ + -p {{ nextcloud_project | quote }} \ + -f {{ nextcloud_compose_file | quote }} \ + pull {{ nextcloud_service | quote }} + + echo "Recreating the Nextcloud container" + {{ docker_cmd }} compose \ + -p {{ nextcloud_project | quote }} \ + -f {{ nextcloud_compose_file | quote }} \ + up -d \ + --no-deps \ + --force-recreate \ {{ nextcloud_service | quote }} - - >- - {{ docker_prefix }} - compose - -p {{ nextcloud_project | quote }} - -f {{ nextcloud_compose_file | quote }} - up -d - --no-deps - --force-recreate - {{ nextcloud_service | quote }} + echo "Waiting for the Nextcloud container" + timeout 180s bash -c ' + until [ "$(docker inspect --format "{% raw %}{{.State.Running}}{% endraw %}" {{ nextcloud_container | quote }} 2>/dev/null)" = "true" ]; do + sleep 3 + done + ' - - >- - timeout 180s bash -c - 'until docker inspect - --format="{{ '{{' }}.State.Running{{ '}}' }}" - {{ nextcloud_container | quote }} - 2>/dev/null | grep -qx true; - do - sleep 3; - done' + echo "Waiting for Nextcloud OCC" + timeout 180s bash -c ' + until docker exec -u www-data {{ nextcloud_container | quote }} php occ status >/dev/null 2>&1; do + sleep 3 + done + ' - - >- - timeout 180s bash -c - 'until docker exec -u www-data - {{ nextcloud_container | quote }} - php occ status >/dev/null 2>&1; - do - sleep 3; - done' - - - >- - {{ docker_prefix }} - exec -u www-data - {{ nextcloud_container }} + echo "Running the Nextcloud database upgrade" + {{ docker_cmd }} exec -u www-data \ + {{ nextcloud_container | quote }} \ php occ upgrade - - >- - {{ docker_prefix }} - exec -u www-data - {{ nextcloud_container }} + echo "Updating Nextcloud applications" + {{ docker_cmd }} exec -u www-data \ + {{ nextcloud_container | quote }} \ php occ app:update --all - - >- - {{ docker_prefix }} - exec -u www-data - {{ nextcloud_container }} + echo "Running Nextcloud maintenance repair" + {{ docker_cmd }} exec -u www-data \ + {{ nextcloud_container | quote }} \ php occ maintenance:repair --include-expensive - - >- - {{ docker_prefix }} - exec -u www-data - {{ nextcloud_container }} + echo "Reading Nextcloud status" + {{ docker_cmd }} exec -u www-data \ + {{ nextcloud_container | quote }} \ php occ status pre_tasks: @@ -172,12 +177,11 @@ - vm_ip | string | trim | length > 0 - vm_user is defined - vm_user | string | trim | length > 0 - - vm_pass is defined - - vm_pass | string | length > 0 + - vm_password | string | length > 0 - retry_count | int > 0 fail_msg: >- Missing or invalid vm_ip, vm_user, vm_pass or RETRIES. - Configure them in the attached Semaphore Variable Group. + Check the attached Semaphore Variable Group. quiet: true no_log: true @@ -188,7 +192,7 @@ state: present update_cache: true - - name: Nextcloud | Show current version before upgrade + - name: Nextcloud | Run preflight checks on VM ansible.builtin.command: argv: - sshpass @@ -199,41 +203,81 @@ - -o - UserKnownHostsFile=/dev/null - -o + - LogLevel=ERROR + - -o + - ConnectTimeout=15 + - -o + - ServerAliveInterval=10 + - -o + - ServerAliveCountMax=3 + - "{{ vm_user }}@{{ vm_ip }}" + - >- + {{ + ('sudo -n bash -lc ' ~ (preflight_script | quote)) + if vm_use_sudo + else + ('bash -lc ' ~ (preflight_script | quote)) + }} + environment: + SSHPASS: "{{ vm_password }}" + register: nc_preflight + changed_when: false + no_log: true + + - name: Nextcloud | Show preflight result + ansible.builtin.debug: + msg: | + RC: {{ nc_preflight.rc }} + + STDOUT: + {{ nc_preflight.stdout | default('') | trim }} + + STDERR: + {{ nc_preflight.stderr | default('') | trim }} + when: debug_level == 1 + + - name: Nextcloud | Read current version before upgrade + ansible.builtin.command: + argv: + - sshpass + - -e + - ssh + - -o + - StrictHostKeyChecking=no + - -o + - UserKnownHostsFile=/dev/null + - -o + - LogLevel=ERROR + - -o - ConnectTimeout=15 - "{{ vm_user }}@{{ vm_ip }}" - >- {{ - ('sudo -n bash -lc ' ~ - (nextcloud_version_command | quote)) + ('sudo -n bash -lc ' ~ (nextcloud_version_command | quote)) if vm_use_sudo else - ('bash -lc ' ~ - (nextcloud_version_command | quote)) + ('bash -lc ' ~ (nextcloud_version_command | quote)) }} environment: - SSHPASS: "{{ vm_pass }}" + SSHPASS: "{{ vm_password }}" register: nc_version_before changed_when: false failed_when: false - when: debug_level == 1 + no_log: true - name: Nextcloud | Print current version ansible.builtin.debug: msg: >- + Current version: {{ nc_version_before.stdout | default('Nextcloud version is unavailable') | trim }} - when: debug_level == 1 - name: Nextcloud | Run backup and upgrade block: - # --------------------------------------------------------------------- - # Backup - # --------------------------------------------------------------------- - - - name: Nextcloud | Run backup commands on VM + - name: Nextcloud | Create configuration and database backup ansible.builtin.command: argv: - sshpass @@ -244,6 +288,8 @@ - -o - UserKnownHostsFile=/dev/null - -o + - LogLevel=ERROR + - -o - ConnectTimeout=15 - -o - ServerAliveInterval=10 @@ -252,60 +298,30 @@ - "{{ vm_user }}@{{ vm_ip }}" - >- {{ - ('sudo -n bash -lc ' ~ (item | quote)) + ('sudo -n bash -lc ' ~ (backup_script | quote)) if vm_use_sudo else - ('bash -lc ' ~ (item | quote)) + ('bash -lc ' ~ (backup_script | quote)) }} environment: - SSHPASS: "{{ vm_pass }}" - loop: "{{ nextcloud_backup_commands }}" - loop_control: - index_var: backup_index - label: "backup-cmd-{{ backup_index }}" - register: nc_backup_cmds - changed_when: false - failed_when: false + SSHPASS: "{{ vm_password }}" + register: nc_backup + no_log: true - - name: Nextcloud | Show backup command outputs + - name: Nextcloud | Show backup output ansible.builtin.debug: msg: | - CMD: {{ item.item }} - RC: {{ item.rc }} + Backup directory: {{ backup_dir }} + RC: {{ nc_backup.rc }} STDOUT: - {{ item.stdout | default('') | trim }} + {{ nc_backup.stdout | default('') | trim }} STDERR: - {{ item.stderr | default('') | trim }} - loop: "{{ nc_backup_cmds.results }}" - loop_control: - index_var: backup_debug_index - label: "backup-cmd-{{ backup_debug_index }}" + {{ nc_backup.stderr | default('') | trim }} when: debug_level == 1 - - name: Nextcloud | Fail if any backup command failed - ansible.builtin.assert: - that: - - item.rc == 0 - fail_msg: | - Nextcloud backup command failed. - Command: {{ item.item }} - RC: {{ item.rc }} - - STDERR: - {{ item.stderr | default('') | trim }} - quiet: true - loop: "{{ nc_backup_cmds.results }}" - loop_control: - index_var: backup_assert_index - label: "backup-cmd-{{ backup_assert_index }}" - - # --------------------------------------------------------------------- - # Upgrade - # --------------------------------------------------------------------- - - - name: Nextcloud | Run upgrade commands on VM + - name: Nextcloud | Pull image and upgrade ansible.builtin.command: argv: - sshpass @@ -316,6 +332,8 @@ - -o - UserKnownHostsFile=/dev/null - -o + - LogLevel=ERROR + - -o - ConnectTimeout=15 - -o - ServerAliveInterval=10 @@ -324,55 +342,28 @@ - "{{ vm_user }}@{{ vm_ip }}" - >- {{ - ('sudo -n bash -lc ' ~ (item | quote)) + ('sudo -n bash -lc ' ~ (upgrade_script | quote)) if vm_use_sudo else - ('bash -lc ' ~ (item | quote)) + ('bash -lc ' ~ (upgrade_script | quote)) }} environment: - SSHPASS: "{{ vm_pass }}" - loop: "{{ nextcloud_upgrade_commands }}" - loop_control: - index_var: upgrade_index - label: "upgrade-cmd-{{ upgrade_index }}" - register: nc_upgrade_cmds - changed_when: false - failed_when: false + SSHPASS: "{{ vm_password }}" + register: nc_upgrade + no_log: true - - name: Nextcloud | Show upgrade command outputs + - name: Nextcloud | Show upgrade output ansible.builtin.debug: msg: | - CMD: {{ item.item }} - RC: {{ item.rc }} + RC: {{ nc_upgrade.rc }} STDOUT: - {{ item.stdout | default('') | trim }} + {{ nc_upgrade.stdout | default('') | trim }} STDERR: - {{ item.stderr | default('') | trim }} - loop: "{{ nc_upgrade_cmds.results }}" - loop_control: - index_var: upgrade_debug_index - label: "upgrade-cmd-{{ upgrade_debug_index }}" + {{ nc_upgrade.stderr | default('') | trim }} when: debug_level == 1 - - name: Nextcloud | Fail if any upgrade command failed - ansible.builtin.assert: - that: - - item.rc == 0 - fail_msg: | - Nextcloud upgrade command failed. - Command: {{ item.item }} - RC: {{ item.rc }} - - STDERR: - {{ item.stderr | default('') | trim }} - quiet: true - loop: "{{ nc_upgrade_cmds.results }}" - loop_control: - index_var: upgrade_assert_index - label: "upgrade-cmd-{{ upgrade_assert_index }}" - always: - name: Nextcloud | Disable maintenance mode ansible.builtin.command: @@ -385,6 +376,8 @@ - -o - UserKnownHostsFile=/dev/null - -o + - LogLevel=ERROR + - -o - ConnectTimeout=15 - "{{ vm_user }}@{{ vm_ip }}" - >- @@ -397,10 +390,11 @@ (nextcloud_maintenance_off_command | quote)) }} environment: - SSHPASS: "{{ vm_pass }}" + SSHPASS: "{{ vm_password }}" register: nc_maint_off changed_when: false failed_when: false + no_log: true - name: Nextcloud | Show maintenance mode result ansible.builtin.debug: @@ -414,7 +408,7 @@ {{ nc_maint_off.stderr | default('') | trim }} when: debug_level == 1 - - name: Nextcloud | Fail if maintenance mode could not be disabled + - name: Nextcloud | Verify maintenance mode was disabled ansible.builtin.assert: that: - nc_maint_off.rc == 0 @@ -426,36 +420,140 @@ {{ nc_maint_off.stderr | default('') | trim }} quiet: true - # ------------------------------------------------------------------------- - # Readiness check - # ------------------------------------------------------------------------- - - - name: Nextcloud | Wait for status.php + - name: Nextcloud | Wait for public status.php ansible.builtin.uri: url: "{{ nextcloud_status_url }}" method: GET return_content: true validate_certs: true status_code: 200 + timeout: 20 register: nc_status delegate_to: localhost run_once: true retries: "{{ retry_count }}" delay: 4 until: - - nc_status.status | default(0) == 200 + - nc_status.status | default(0) | int == 200 - nc_status.json is defined - nc_status.json.installed | default(false) | bool - not (nc_status.json.maintenance | default(true) | bool) - not (nc_status.json.needsDbUpgrade | default(true) | bool) changed_when: false + - name: Nextcloud | Check required containers after upgrade + ansible.builtin.command: + argv: + - sshpass + - -e + - ssh + - -o + - StrictHostKeyChecking=no + - -o + - UserKnownHostsFile=/dev/null + - -o + - LogLevel=ERROR + - -o + - ConnectTimeout=15 + - "{{ vm_user }}@{{ vm_ip }}" + - >- + {{ + ( + 'sudo -n bash -lc ' ~ + ( + 'for container in ' ~ + (nextcloud_container | quote) ~ ' ' ~ + (nextcloud_db_container | quote) ~ ' ' ~ + (redis_container | quote) ~ + '; do state="$(docker inspect --format ' ~ + ("'{{.State.Running}}'") ~ + ' "${container}" 2>/dev/null)"; ' ~ + '[ "${state}" = "true" ] || exit 1; done' + | quote + ) + ) + if vm_use_sudo + else + ( + 'bash -lc ' ~ + ( + 'for container in ' ~ + (nextcloud_container | quote) ~ ' ' ~ + (nextcloud_db_container | quote) ~ ' ' ~ + (redis_container | quote) ~ + '; do state="$(docker inspect --format ' ~ + ("'{{.State.Running}}'") ~ + ' "${container}" 2>/dev/null)"; ' ~ + '[ "${state}" = "true" ] || exit 1; done' + | quote + ) + ) + }} + environment: + SSHPASS: "{{ vm_password }}" + register: nc_containers_after + changed_when: false + no_log: true + + - name: Nextcloud | Check external Collabora root endpoint + ansible.builtin.uri: + url: "{{ collabora_url }}" + method: GET + return_content: true + validate_certs: true + status_code: 200 + timeout: 20 + register: collabora_root + delegate_to: localhost + run_once: true + changed_when: false + failed_when: >- + (collabora_root.status | default(0) | int != 200) + or + ('OK' not in (collabora_root.content | default(''))) + + - name: Nextcloud | Check external Collabora discovery endpoint + ansible.builtin.uri: + url: "{{ collabora_discovery_url }}" + method: GET + return_content: true + validate_certs: true + status_code: 200 + timeout: 20 + register: collabora_discovery + delegate_to: localhost + run_once: true + changed_when: false + failed_when: >- + (collabora_discovery.status | default(0) | int != 200) + or + ('- - Nextcloud {{ nc_status.json.version | default('?') }} - (installed={{ nc_status.json.installed | default('?') }}, - maintenance={{ nc_status.json.maintenance | default('?') }}, - needsDbUpgrade={{ - nc_status.json.needsDbUpgrade | default('?') - }}) \ No newline at end of file + msg: | + Nextcloud update completed successfully. + + Backup directory: + {{ backup_dir }} + + Nextcloud version: + {{ nc_status.json.version | default('?') }} + + Installed: + {{ nc_status.json.installed | default('?') }} + + Maintenance: + {{ nc_status.json.maintenance | default('?') }} + + Needs database upgrade: + {{ nc_status.json.needsDbUpgrade | default('?') }} + + Nextcloud container: OK + MariaDB container: OK + Redis container: OK + Nextcloud public status endpoint: OK + Collabora root endpoint: OK + Collabora discovery endpoint: OK + run_once: true