From aa9bdb47dfbc9cfc2459977569ea61c876ef6960 Mon Sep 17 00:00:00 2001 From: "martin.fencl" Date: Thu, 6 Aug 2026 18:34:31 +0200 Subject: [PATCH] fix: enhance Nextcloud update playbook with improved preflight checks, backup logic, and upgrade process --- nextcloud/update_nextcloud_v2.yml | 334 +++++++++++++++++++++++------- 1 file changed, 263 insertions(+), 71 deletions(-) diff --git a/nextcloud/update_nextcloud_v2.yml b/nextcloud/update_nextcloud_v2.yml index 3dfae8f..7717779 100644 --- a/nextcloud/update_nextcloud_v2.yml +++ b/nextcloud/update_nextcloud_v2.yml @@ -11,26 +11,29 @@ # vm_pass is stored in Semaphore Secrets as an environment variable. vm_password: "{{ lookup('ansible.builtin.env', 'vm_pass') }}" - # Values from Semaphore Extra variables. + # Values stored in Semaphore Extra variables. vm_use_sudo: "{{ use_sudo | default(false) | bool }}" debug_level: "{{ DEBUG | default(0) | int }}" retry_count: "{{ RETRIES | default(25) | int }}" - # Nextcloud configuration. + # Docker Compose configuration. nextcloud_project: "nextcloud-collabora" nextcloud_compose_file: "/data/compose/nextcloud/docker-compose-nextcloud.yml" nextcloud_service: "nextcloud" + # Container names. nextcloud_container: "nextcloud" nextcloud_db_container: "nextcloud-db" redis_container: "redis" + # Public endpoints. nextcloud_base_url: "https://cloud.martinfencl.eu" nextcloud_status_url: "{{ nextcloud_base_url }}/status.php" collabora_url: "https://collabora.martinfencl.eu/" collabora_discovery_url: "https://collabora.martinfencl.eu/hosting/discovery" + # Backup configuration. backup_timestamp: "{{ lookup('pipe', 'date +%F-%H%M%S') }}" backup_dir: "/data/compose/nextcloud/backup-{{ backup_timestamp }}" @@ -46,15 +49,22 @@ {{ nextcloud_container | quote }} php occ maintenance:mode --off + # ------------------------------------------------------------------------- + # Preflight + # ------------------------------------------------------------------------- + preflight_script: | set -euo pipefail + echo "Checking Docker" command -v docker >/dev/null docker version >/dev/null docker compose version >/dev/null + echo "Checking Docker Compose file" test -f {{ nextcloud_compose_file | quote }} + echo "Checking required containers" for container in \ {{ nextcloud_container | quote }} \ {{ nextcloud_db_container | quote }} \ @@ -68,33 +78,78 @@ fi done + echo "Checking Nextcloud status" {{ docker_cmd }} exec -u www-data \ {{ nextcloud_container | quote }} \ php occ status - backup_script: | - set -euo pipefail + echo "Checking MariaDB" + {{ docker_cmd }} exec \ + {{ nextcloud_db_container | quote }} \ + sh -c ' + mariadb-admin ping -h 127.0.0.1 --silent 2>/dev/null || + mysqladmin ping -h 127.0.0.1 --silent 2>/dev/null + ' + echo "Checking Redis" + {{ docker_cmd }} exec \ + {{ redis_container | quote }} \ + redis-cli -h 127.0.0.1 ping | + grep -qx PONG + + echo "Preflight checks completed successfully" + + # ------------------------------------------------------------------------- + # Backup + # ------------------------------------------------------------------------- + + backup_script: | + set -Eeuo pipefail + + current_step="backup initialization" + + trap ' + rc=$? + echo >&2 + echo "Backup failed" >&2 + echo "Step: ${current_step}" >&2 + echo "Line: ${LINENO}" >&2 + echo "Command: ${BASH_COMMAND}" >&2 + echo "Return code: ${rc}" >&2 + exit "${rc}" + ' ERR + + current_step="creating backup directory" + echo "Creating backup directory: {{ backup_dir }}" mkdir -p {{ backup_dir | quote }} + current_step="enabling maintenance mode" echo "Enabling Nextcloud maintenance mode" {{ docker_cmd }} exec -u www-data \ {{ nextcloud_container | quote }} \ php occ maintenance:mode --on + current_step="creating configuration archive" echo "Backing up Nextcloud configuration and custom applications" {{ docker_cmd }} exec \ {{ nextcloud_container | quote }} \ - sh -c 'tar czf /tmp/nextcloud_conf.tgz -C /var/www/html config custom_apps' + sh -c ' + tar czf /tmp/nextcloud_conf.tgz \ + -C /var/www/html \ + config custom_apps + ' + current_step="copying configuration archive" {{ docker_cmd }} cp \ {{ nextcloud_container | quote }}:/tmp/nextcloud_conf.tgz \ {{ (backup_dir ~ '/nextcloud_conf.tgz') | quote }} + current_step="removing temporary configuration archive" {{ docker_cmd }} exec \ {{ nextcloud_container | quote }} \ rm -f /tmp/nextcloud_conf.tgz + current_step="creating database dump" echo "Backing up Nextcloud database" {{ docker_cmd }} exec \ {{ nextcloud_db_container | quote }} \ @@ -112,20 +167,50 @@ fi ' > {{ (backup_dir ~ '/db.sql') | quote }} + current_step="validating configuration archive" test -s {{ (backup_dir ~ '/nextcloud_conf.tgz') | quote }} + + current_step="validating database dump" test -s {{ (backup_dir ~ '/db.sql') | quote }} - echo "Backup completed successfully: {{ backup_dir }}" + echo "Backup completed successfully" + echo "Backup directory: {{ backup_dir }}" + + # ------------------------------------------------------------------------- + # Upgrade + # ------------------------------------------------------------------------- upgrade_script: | - set -euo pipefail + set -Eeuo pipefail + current_step="upgrade initialization" + + trap ' + rc=$? + echo >&2 + echo "Nextcloud upgrade failed" >&2 + echo "Step: ${current_step}" >&2 + echo "Line: ${LINENO}" >&2 + echo "Command: ${BASH_COMMAND}" >&2 + echo "Return code: ${rc}" >&2 + exit "${rc}" + ' ERR + + current_step="docker compose config validation" + echo "Validating Docker Compose configuration" + {{ docker_cmd }} compose \ + -p {{ nextcloud_project | quote }} \ + -f {{ nextcloud_compose_file | quote }} \ + config --quiet + + current_step="docker compose pull" echo "Pulling the current Nextcloud image" {{ docker_cmd }} compose \ -p {{ nextcloud_project | quote }} \ -f {{ nextcloud_compose_file | quote }} \ pull {{ nextcloud_service | quote }} + current_step="docker compose up" echo "Recreating the Nextcloud container" {{ docker_cmd }} compose \ -p {{ nextcloud_project | quote }} \ @@ -135,40 +220,118 @@ --force-recreate \ {{ nextcloud_service | quote }} + current_step="waiting for the Nextcloud container" echo "Waiting for the Nextcloud container" - timeout 180s bash -c ' - until [ "$(docker inspect --format "{% raw %}{{.State.Running}}{% endraw %}" {{ nextcloud_container | quote }} 2>/dev/null)" = "true" ]; do - sleep 3 - done - ' + container_deadline=$((SECONDS + 180)) + + while true + do + container_state="$(docker inspect --format '{% raw %}{{.State.Running}}{% endraw %}' {{ nextcloud_container | quote }} 2>/dev/null || true)" + + if [ "${container_state}" = "true" ]; then + break + fi + + if [ "${SECONDS}" -ge "${container_deadline}" ]; then + echo "Timeout waiting for container {{ nextcloud_container }}" >&2 + exit 124 + fi + + sleep 3 + done + + current_step="waiting for Nextcloud OCC" echo "Waiting for Nextcloud OCC" - timeout 180s bash -c ' - until docker exec -u www-data {{ nextcloud_container | quote }} php occ status >/dev/null 2>&1; do - sleep 3 - done - ' + occ_deadline=$((SECONDS + 180)) + + while true + do + if docker exec -u www-data \ + {{ nextcloud_container | quote }} \ + php occ status >/dev/null 2>&1 + then + break + fi + + if [ "${SECONDS}" -ge "${occ_deadline}" ]; then + echo "Timeout waiting for Nextcloud OCC" >&2 + exit 124 + fi + + sleep 3 + done + + current_step="Nextcloud database upgrade" echo "Running the Nextcloud database upgrade" {{ docker_cmd }} exec -u www-data \ {{ nextcloud_container | quote }} \ php occ upgrade + current_step="Nextcloud application update" echo "Updating Nextcloud applications" {{ docker_cmd }} exec -u www-data \ {{ nextcloud_container | quote }} \ php occ app:update --all + current_step="Nextcloud maintenance repair" echo "Running Nextcloud maintenance repair" {{ docker_cmd }} exec -u www-data \ {{ nextcloud_container | quote }} \ php occ maintenance:repair --include-expensive + current_step="reading final Nextcloud status" echo "Reading Nextcloud status" {{ docker_cmd }} exec -u www-data \ {{ nextcloud_container | quote }} \ php occ status + echo "Nextcloud upgrade commands completed successfully" + + # ------------------------------------------------------------------------- + # Post-upgrade checks + # ------------------------------------------------------------------------- + + postcheck_script: | + set -euo pipefail + + echo "Checking required containers after upgrade" + + for container in \ + {{ nextcloud_container | quote }} \ + {{ nextcloud_db_container | quote }} \ + {{ redis_container | quote }} + do + state="$(docker inspect --format '{% raw %}{{.State.Running}}{% endraw %}' "${container}" 2>/dev/null || true)" + + if [ "${state}" != "true" ]; then + echo "Container ${container} is not running" >&2 + exit 1 + fi + done + + echo "Checking MariaDB after upgrade" + {{ docker_cmd }} exec \ + {{ nextcloud_db_container | quote }} \ + sh -c ' + mariadb-admin ping -h 127.0.0.1 --silent 2>/dev/null || + mysqladmin ping -h 127.0.0.1 --silent 2>/dev/null + ' + + echo "Checking Redis after upgrade" + {{ docker_cmd }} exec \ + {{ redis_container | quote }} \ + redis-cli -h 127.0.0.1 ping | + grep -qx PONG + + echo "Checking Nextcloud status after upgrade" + {{ docker_cmd }} exec -u www-data \ + {{ nextcloud_container | quote }} \ + php occ status + + echo "Post-upgrade checks completed successfully" + pre_tasks: - name: Validate VM connection variables ansible.builtin.assert: @@ -192,6 +355,10 @@ state: present update_cache: true + # ------------------------------------------------------------------------- + # Preflight + # ------------------------------------------------------------------------- + - name: Nextcloud | Run preflight checks on VM ansible.builtin.command: argv: @@ -227,7 +394,7 @@ - name: Nextcloud | Show preflight result ansible.builtin.debug: msg: | - RC: {{ nc_preflight.rc }} + Preflight return code: {{ nc_preflight.rc }} STDOUT: {{ nc_preflight.stdout | default('') | trim }} @@ -236,6 +403,10 @@ {{ nc_preflight.stderr | default('') | trim }} when: debug_level == 1 + # ------------------------------------------------------------------------- + # Current version + # ------------------------------------------------------------------------- + - name: Nextcloud | Read current version before upgrade ansible.builtin.command: argv: @@ -250,13 +421,19 @@ - LogLevel=ERROR - -o - ConnectTimeout=15 + - -o + - ServerAliveInterval=10 + - -o + - ServerAliveCountMax=3 - "{{ vm_user }}@{{ vm_ip }}" - >- {{ - ('sudo -n bash -lc ' ~ (nextcloud_version_command | quote)) + ('sudo -n bash -lc ' ~ + (nextcloud_version_command | quote)) if vm_use_sudo else - ('bash -lc ' ~ (nextcloud_version_command | quote)) + ('bash -lc ' ~ + (nextcloud_version_command | quote)) }} environment: SSHPASS: "{{ vm_password }}" @@ -275,6 +452,10 @@ | trim }} + # ------------------------------------------------------------------------- + # Backup and upgrade + # ------------------------------------------------------------------------- + - name: Nextcloud | Run backup and upgrade block: - name: Nextcloud | Create configuration and database backup @@ -306,13 +487,14 @@ environment: SSHPASS: "{{ vm_password }}" register: nc_backup + changed_when: true no_log: true - name: Nextcloud | Show backup output ansible.builtin.debug: msg: | Backup directory: {{ backup_dir }} - RC: {{ nc_backup.rc }} + Backup return code: {{ nc_backup.rc }} STDOUT: {{ nc_backup.stdout | default('') | trim }} @@ -321,7 +503,7 @@ {{ nc_backup.stderr | default('') | trim }} when: debug_level == 1 - - name: Nextcloud | Pull image and upgrade + - name: Nextcloud | Pull image and run upgrade ansible.builtin.command: argv: - sshpass @@ -382,18 +564,6 @@ {{ nc_upgrade.stderr | default('') | trim }} quiet: true - - name: Nextcloud | Show upgrade output - ansible.builtin.debug: - msg: | - RC: {{ nc_upgrade.rc }} - - STDOUT: - {{ nc_upgrade.stdout | default('') | trim }} - - STDERR: - {{ nc_upgrade.stderr | default('') | trim }} - when: debug_level == 1 - always: - name: Nextcloud | Disable maintenance mode ansible.builtin.command: @@ -409,6 +579,10 @@ - LogLevel=ERROR - -o - ConnectTimeout=15 + - -o + - ServerAliveInterval=10 + - -o + - ServerAliveCountMax=3 - "{{ vm_user }}@{{ vm_ip }}" - >- {{ @@ -429,14 +603,15 @@ - name: Nextcloud | Show maintenance mode result ansible.builtin.debug: msg: | - RC: {{ nc_maint_off.rc }} + Maintenance return code: {{ nc_maint_off.rc }} STDOUT: {{ nc_maint_off.stdout | default('') | trim }} STDERR: {{ nc_maint_off.stderr | default('') | trim }} - when: debug_level == 1 + when: + - debug_level == 1 or nc_maint_off.rc != 0 - name: Nextcloud | Verify maintenance mode was disabled ansible.builtin.assert: @@ -444,12 +619,21 @@ - nc_maint_off.rc == 0 fail_msg: | Failed to disable Nextcloud maintenance mode. - RC: {{ nc_maint_off.rc }} + + Return code: + {{ nc_maint_off.rc }} + + STDOUT: + {{ nc_maint_off.stdout | default('') | trim }} STDERR: {{ nc_maint_off.stderr | default('') | trim }} quiet: true + # ------------------------------------------------------------------------- + # Public Nextcloud readiness + # ------------------------------------------------------------------------- + - name: Nextcloud | Wait for public status.php ansible.builtin.uri: url: "{{ nextcloud_status_url }}" @@ -471,7 +655,11 @@ - not (nc_status.json.needsDbUpgrade | default(true) | bool) changed_when: false - - name: Nextcloud | Check required containers after upgrade + # ------------------------------------------------------------------------- + # Post-upgrade checks + # ------------------------------------------------------------------------- + + - name: Nextcloud | Run post-upgrade checks on VM ansible.builtin.command: argv: - sshpass @@ -485,46 +673,40 @@ - LogLevel=ERROR - -o - ConnectTimeout=15 + - -o + - ServerAliveInterval=10 + - -o + - ServerAliveCountMax=3 - "{{ vm_user }}@{{ vm_ip }}" - >- {{ - ( - 'sudo -n bash -lc ' ~ - ( - 'for container in ' ~ - (nextcloud_container | quote) ~ ' ' ~ - (nextcloud_db_container | quote) ~ ' ' ~ - (redis_container | quote) ~ - '; do state="$(docker inspect --format ' ~ - ("'{{.State.Running}}'") ~ - ' "${container}" 2>/dev/null)"; ' ~ - '[ "${state}" = "true" ] || exit 1; done' - | quote - ) - ) + ('sudo -n bash -lc ' ~ (postcheck_script | quote)) if vm_use_sudo else - ( - 'bash -lc ' ~ - ( - 'for container in ' ~ - (nextcloud_container | quote) ~ ' ' ~ - (nextcloud_db_container | quote) ~ ' ' ~ - (redis_container | quote) ~ - '; do state="$(docker inspect --format ' ~ - ("'{{.State.Running}}'") ~ - ' "${container}" 2>/dev/null)"; ' ~ - '[ "${state}" = "true" ] || exit 1; done' - | quote - ) - ) + ('bash -lc ' ~ (postcheck_script | quote)) }} environment: SSHPASS: "{{ vm_password }}" - register: nc_containers_after + register: nc_postcheck changed_when: false no_log: true + - name: Nextcloud | Show post-upgrade checks + ansible.builtin.debug: + msg: | + Post-upgrade return code: {{ nc_postcheck.rc }} + + STDOUT: + {{ nc_postcheck.stdout | default('') | trim }} + + STDERR: + {{ nc_postcheck.stderr | default('') | trim }} + when: debug_level == 1 + + # ------------------------------------------------------------------------- + # External Collabora + # ------------------------------------------------------------------------- + - name: Nextcloud | Check external Collabora root endpoint ansible.builtin.uri: url: "{{ collabora_url }}" @@ -560,17 +742,25 @@ ('