fix: enhance Nextcloud update playbook with improved preflight checks, backup logic, and error handling

nextcloud/update_nextcloud_v2.yml

@@ -31,12 +31,13 @@
     nextcloud_status_url: "{{ nextcloud_base_url }}/status.php"
 
     collabora_url: "https://collabora.martinfencl.eu/"
-    collabora_discovery_url: "https://collabora.martinfencl.eu/hosting/discovery"
+    collabora_discovery_url: >-
+      https://collabora.martinfencl.eu/hosting/discovery
 
     # Backup configuration.
-    backup_timestamp: "{{ lookup('pipe', 'date +%F-%H%M%S') }}"
+    backup_root: "/data/compose/nextcloud"
-    backup_dir: "/data/compose/nextcloud/backup-{{ backup_timestamp }}"
 
+    # Docker command without CLI hints.
     docker_cmd: "env DOCKER_CLI_HINTS=0 docker"
 
     nextcloud_version_command: >-
@@ -50,21 +51,51 @@
       php occ maintenance:mode --off
 
     # -------------------------------------------------------------------------
-    # Preflight
+    # Preflight script
     # -------------------------------------------------------------------------
 
     preflight_script: |
-      set -euo pipefail
+      set -Eeuo pipefail
 
-      echo "Checking Docker"
+      current_step="preflight initialization"
+
+      trap '
+        rc=$?
+        echo >&2
+        echo "Preflight check failed" >&2
+        echo "Step: ${current_step}" >&2
+        echo "Line: ${LINENO}" >&2
+        echo "Command: ${BASH_COMMAND}" >&2
+        echo "Return code: ${rc}" >&2
+        exit "${rc}"
+      ' ERR
+
+      current_step="checking Docker executable"
+      echo "Checking Docker executable"
       command -v docker >/dev/null
+
+      current_step="checking Docker daemon"
+      echo "Checking Docker daemon"
       docker version >/dev/null
+
+      current_step="checking Docker Compose"
+      echo "Checking Docker Compose"
       docker compose version >/dev/null
 
+      current_step="checking Docker Compose file"
       echo "Checking Docker Compose file"
       test -f {{ nextcloud_compose_file | quote }}
 
+      current_step="validating Docker Compose configuration"
+      echo "Validating Docker Compose configuration"
+      {{ docker_cmd }} compose \
+        -p {{ nextcloud_project | quote }} \
+        -f {{ nextcloud_compose_file | quote }} \
+        config --quiet
+
+      current_step="checking required containers"
       echo "Checking required containers"
+
       for container in \
         {{ nextcloud_container | quote }} \
         {{ nextcloud_db_container | quote }} \
@@ -78,19 +109,29 @@
         fi
       done
 
+      current_step="checking Nextcloud status"
       echo "Checking Nextcloud status"
       {{ docker_cmd }} exec -u www-data \
         {{ nextcloud_container | quote }} \
         php occ status
 
+      current_step="checking MariaDB"
       echo "Checking MariaDB"
       {{ docker_cmd }} exec \
         {{ nextcloud_db_container | quote }} \
         sh -c '
-          mariadb-admin ping -h 127.0.0.1 --silent 2>/dev/null ||
+          if command -v mariadb-admin >/dev/null 2>&1; then
-          mysqladmin ping -h 127.0.0.1 --silent 2>/dev/null
+            exec mariadb-admin ping \
+              -h 127.0.0.1 \
+              --silent
+          else
+            exec mysqladmin ping \
+              -h 127.0.0.1 \
+              --silent
+          fi
         '
 
+      current_step="checking Redis"
       echo "Checking Redis"
       {{ docker_cmd }} exec \
         {{ redis_container | quote }} \
@@ -100,7 +141,7 @@
       echo "Preflight checks completed successfully"
 
     # -------------------------------------------------------------------------
-    # Backup
+    # Backup script
     # -------------------------------------------------------------------------
 
     backup_script: |
@@ -111,7 +152,7 @@
       trap '
         rc=$?
         echo >&2
-        echo "Backup failed" >&2
+        echo "Nextcloud backup failed" >&2
         echo "Step: ${current_step}" >&2
         echo "Line: ${LINENO}" >&2
         echo "Command: ${BASH_COMMAND}" >&2
@@ -140,11 +181,13 @@
         '
 
       current_step="copying configuration archive"
+      echo "Copying configuration archive"
       {{ docker_cmd }} cp \
         {{ nextcloud_container | quote }}:/tmp/nextcloud_conf.tgz \
         {{ (backup_dir ~ '/nextcloud_conf.tgz') | quote }}
 
       current_step="removing temporary configuration archive"
+      echo "Removing temporary configuration archive"
       {{ docker_cmd }} exec \
         {{ nextcloud_container | quote }} \
         rm -f /tmp/nextcloud_conf.tgz
@@ -168,16 +211,40 @@
         ' > {{ (backup_dir ~ '/db.sql') | quote }}
 
       current_step="validating configuration archive"
+      echo "Validating configuration archive"
       test -s {{ (backup_dir ~ '/nextcloud_conf.tgz') | quote }}
 
       current_step="validating database dump"
+      echo "Validating database dump"
       test -s {{ (backup_dir ~ '/db.sql') | quote }}
 
+      current_step="disabling maintenance mode after backup"
+      echo "Disabling Nextcloud maintenance mode after backup"
+      {{ docker_cmd }} exec -u www-data \
+        {{ nextcloud_container | quote }} \
+        php occ maintenance:mode --off
+
+      current_step="verifying maintenance mode after backup"
+      echo "Verifying maintenance mode after backup"
+
+      maintenance_state="$(
+        {{ docker_cmd }} exec -u www-data \
+          {{ nextcloud_container | quote }} \
+          php occ config:system:get maintenance \
+          2>/dev/null || true
+      )"
+
+      if [ "${maintenance_state}" = "true" ] || \
+         [ "${maintenance_state}" = "1" ]; then
+        echo "Maintenance mode is still enabled after backup" >&2
+        exit 1
+      fi
+
       echo "Backup completed successfully"
       echo "Backup directory: {{ backup_dir }}"
 
     # -------------------------------------------------------------------------
-    # Upgrade
+    # Upgrade script
     # -------------------------------------------------------------------------
 
     upgrade_script: |
@@ -196,21 +263,21 @@
         exit "${rc}"
       ' ERR
 
-      current_step="docker compose config validation"
+      current_step="validating Docker Compose configuration"
       echo "Validating Docker Compose configuration"
       {{ docker_cmd }} compose \
         -p {{ nextcloud_project | quote }} \
         -f {{ nextcloud_compose_file | quote }} \
         config --quiet
 
-      current_step="docker compose pull"
+      current_step="pulling Nextcloud image"
       echo "Pulling the current Nextcloud image"
       {{ docker_cmd }} compose \
         -p {{ nextcloud_project | quote }} \
         -f {{ nextcloud_compose_file | quote }} \
         pull {{ nextcloud_service | quote }}
 
-      current_step="docker compose up"
+      current_step="recreating Nextcloud container"
       echo "Recreating the Nextcloud container"
       {{ docker_cmd }} compose \
         -p {{ nextcloud_project | quote }} \
@@ -220,7 +287,7 @@
         --force-recreate \
         {{ nextcloud_service | quote }}
 
-      current_step="waiting for the Nextcloud container"
+      current_step="waiting for Nextcloud container"
       echo "Waiting for the Nextcloud container"
 
       container_deadline=$((SECONDS + 180))
@@ -248,7 +315,7 @@
 
       while true
       do
-        if docker exec -u www-data \
+        if {{ docker_cmd }} exec -u www-data \
           {{ nextcloud_container | quote }} \
           php occ status >/dev/null 2>&1
         then
@@ -263,19 +330,51 @@
         sleep 3
       done
 
-      current_step="Nextcloud database upgrade"
+      current_step="disabling stale maintenance mode"
+      echo "Ensuring maintenance mode is disabled before OCC upgrade"
+
+      maintenance_state="$(
+        {{ docker_cmd }} exec -u www-data \
+          {{ nextcloud_container | quote }} \
+          php occ config:system:get maintenance \
+          2>/dev/null || true
+      )"
+
+      if [ "${maintenance_state}" = "true" ] || \
+         [ "${maintenance_state}" = "1" ]; then
+        echo "Maintenance mode is enabled; disabling it"
+        {{ docker_cmd }} exec -u www-data \
+          {{ nextcloud_container | quote }} \
+          php occ maintenance:mode --off
+      fi
+
+      current_step="verifying maintenance mode before OCC upgrade"
+      maintenance_state="$(
+        {{ docker_cmd }} exec -u www-data \
+          {{ nextcloud_container | quote }} \
+          php occ config:system:get maintenance \
+          2>/dev/null || true
+      )"
+
+      if [ "${maintenance_state}" = "true" ] || \
+         [ "${maintenance_state}" = "1" ]; then
+        echo "Maintenance mode could not be disabled" >&2
+        exit 1
+      fi
+
+      current_step="running Nextcloud database upgrade"
       echo "Running the Nextcloud database upgrade"
       {{ docker_cmd }} exec -u www-data \
         {{ nextcloud_container | quote }} \
-        php occ upgrade
+        php occ upgrade --no-interaction
 
-      current_step="Nextcloud application update"
+      current_step="updating Nextcloud applications"
       echo "Updating Nextcloud applications"
       {{ docker_cmd }} exec -u www-data \
         {{ nextcloud_container | quote }} \
         php occ app:update --all
 
-      current_step="Nextcloud maintenance repair"
+      current_step="running Nextcloud maintenance repair"
       echo "Running Nextcloud maintenance repair"
       {{ docker_cmd }} exec -u www-data \
         {{ nextcloud_container | quote }} \
@@ -290,12 +389,26 @@
       echo "Nextcloud upgrade commands completed successfully"
 
     # -------------------------------------------------------------------------
-    # Post-upgrade checks
+    # Post-upgrade script
     # -------------------------------------------------------------------------
 
     postcheck_script: |
-      set -euo pipefail
+      set -Eeuo pipefail
 
+      current_step="post-upgrade initialization"
+
+      trap '
+        rc=$?
+        echo >&2
+        echo "Post-upgrade check failed" >&2
+        echo "Step: ${current_step}" >&2
+        echo "Line: ${LINENO}" >&2
+        echo "Command: ${BASH_COMMAND}" >&2
+        echo "Return code: ${rc}" >&2
+        exit "${rc}"
+      ' ERR
+
+      current_step="checking required containers"
       echo "Checking required containers after upgrade"
 
       for container in \
@@ -311,25 +424,49 @@
         fi
       done
 
+      current_step="checking MariaDB"
       echo "Checking MariaDB after upgrade"
       {{ docker_cmd }} exec \
         {{ nextcloud_db_container | quote }} \
         sh -c '
-          mariadb-admin ping -h 127.0.0.1 --silent 2>/dev/null ||
+          if command -v mariadb-admin >/dev/null 2>&1; then
-          mysqladmin ping -h 127.0.0.1 --silent 2>/dev/null
+            exec mariadb-admin ping \
+              -h 127.0.0.1 \
+              --silent
+          else
+            exec mysqladmin ping \
+              -h 127.0.0.1 \
+              --silent
+          fi
         '
 
+      current_step="checking Redis"
       echo "Checking Redis after upgrade"
       {{ docker_cmd }} exec \
         {{ redis_container | quote }} \
         redis-cli -h 127.0.0.1 ping |
         grep -qx PONG
 
+      current_step="checking Nextcloud status"
       echo "Checking Nextcloud status after upgrade"
       {{ docker_cmd }} exec -u www-data \
         {{ nextcloud_container | quote }} \
         php occ status
 
+      current_step="checking Nextcloud maintenance state"
+      maintenance_state="$(
+        {{ docker_cmd }} exec -u www-data \
+          {{ nextcloud_container | quote }} \
+          php occ config:system:get maintenance \
+          2>/dev/null || true
+      )"
+
+      if [ "${maintenance_state}" = "true" ] || \
+         [ "${maintenance_state}" = "1" ]; then
+        echo "Nextcloud is still in maintenance mode" >&2
+        exit 1
+      fi
+
       echo "Post-upgrade checks completed successfully"
 
   pre_tasks:
@@ -348,6 +485,17 @@
         quiet: true
       no_log: true
 
+    - name: Generate backup timestamp
+      ansible.builtin.set_fact:
+        backup_timestamp: "{{ now(utc=true, fmt='%Y-%m-%d-%H%M%S') }}"
+      changed_when: false
+
+    - name: Set backup directory
+      ansible.builtin.set_fact:
+        backup_dir: >-
+          {{ backup_root }}/backup-{{ backup_timestamp }}
+      changed_when: false
+
   tasks:
     - name: Ensure sshpass is installed
       ansible.builtin.apt:
@@ -389,6 +537,7 @@
         SSHPASS: "{{ vm_password }}"
       register: nc_preflight
       changed_when: false
+      failed_when: false
       no_log: true
 
     - name: Nextcloud | Show preflight result
@@ -401,7 +550,25 @@
 
           STDERR:
           {{ nc_preflight.stderr | default('') | trim }}
-      when: debug_level == 1
+      when:
+        - debug_level == 1 or nc_preflight.rc != 0
+
+    - name: Nextcloud | Fail when preflight checks failed
+      ansible.builtin.assert:
+        that:
+          - nc_preflight.rc == 0
+        fail_msg: |
+          Nextcloud preflight checks failed.
+
+          Return code:
+          {{ nc_preflight.rc }}
+
+          STDOUT:
+          {{ nc_preflight.stdout | default('') | trim }}
+
+          STDERR:
+          {{ nc_preflight.stderr | default('') | trim }}
+        quiet: true
 
     # -------------------------------------------------------------------------
     # Current version
@@ -487,7 +654,8 @@
           environment:
             SSHPASS: "{{ vm_password }}"
           register: nc_backup
-          changed_when: true
+          changed_when: nc_backup.rc == 0
+          failed_when: false
           no_log: true
 
         - name: Nextcloud | Show backup output
@@ -501,7 +669,28 @@
 
               STDERR:
               {{ nc_backup.stderr | default('') | trim }}
-          when: debug_level == 1
+          when:
+            - debug_level == 1 or nc_backup.rc != 0
+
+        - name: Nextcloud | Fail when backup failed
+          ansible.builtin.assert:
+            that:
+              - nc_backup.rc == 0
+            fail_msg: |
+              Nextcloud backup failed.
+
+              Backup directory:
+              {{ backup_dir }}
+
+              Return code:
+              {{ nc_backup.rc }}
+
+              STDOUT:
+              {{ nc_backup.stdout | default('') | trim }}
+
+              STDERR:
+              {{ nc_backup.stderr | default('') | trim }}
+            quiet: true
 
         - name: Nextcloud | Pull image and run upgrade
           ansible.builtin.command:
@@ -689,6 +878,7 @@
         SSHPASS: "{{ vm_password }}"
       register: nc_postcheck
       changed_when: false
+      failed_when: false
       no_log: true
 
     - name: Nextcloud | Show post-upgrade checks
@@ -701,10 +891,28 @@
 
           STDERR:
           {{ nc_postcheck.stderr | default('') | trim }}
-      when: debug_level == 1
+      when:
+        - debug_level == 1 or nc_postcheck.rc != 0
+
+    - name: Nextcloud | Fail when post-upgrade checks failed
+      ansible.builtin.assert:
+        that:
+          - nc_postcheck.rc == 0
+        fail_msg: |
+          Nextcloud post-upgrade checks failed.
+
+          Return code:
+          {{ nc_postcheck.rc }}
+
+          STDOUT:
+          {{ nc_postcheck.stdout | default('') | trim }}
+
+          STDERR:
+          {{ nc_postcheck.stderr | default('') | trim }}
+        quiet: true
 
     # -------------------------------------------------------------------------
-    # External Collabora
+    # External Collabora checks
     # -------------------------------------------------------------------------
 
     - name: Nextcloud | Check external Collabora root endpoint