fix: enhance Nextcloud update playbook with improved preflight checks, backup logic, and upgrade process

nextcloud/update_nextcloud_v2.yml

@@ -11,26 +11,29 @@
     # vm_pass is stored in Semaphore Secrets as an environment variable.
     vm_password: "{{ lookup('ansible.builtin.env', 'vm_pass') }}"
 
-    # Values from Semaphore Extra variables.
+    # Values stored in Semaphore Extra variables.
     vm_use_sudo: "{{ use_sudo | default(false) | bool }}"
     debug_level: "{{ DEBUG | default(0) | int }}"
     retry_count: "{{ RETRIES | default(25) | int }}"
 
-    # Nextcloud configuration.
+    # Docker Compose configuration.
     nextcloud_project: "nextcloud-collabora"
     nextcloud_compose_file: "/data/compose/nextcloud/docker-compose-nextcloud.yml"
     nextcloud_service: "nextcloud"
 
+    # Container names.
     nextcloud_container: "nextcloud"
     nextcloud_db_container: "nextcloud-db"
     redis_container: "redis"
 
+    # Public endpoints.
     nextcloud_base_url: "https://cloud.martinfencl.eu"
     nextcloud_status_url: "{{ nextcloud_base_url }}/status.php"
 
     collabora_url: "https://collabora.martinfencl.eu/"
     collabora_discovery_url: "https://collabora.martinfencl.eu/hosting/discovery"
 
+    # Backup configuration.
     backup_timestamp: "{{ lookup('pipe', 'date +%F-%H%M%S') }}"
     backup_dir: "/data/compose/nextcloud/backup-{{ backup_timestamp }}"
 
@@ -46,15 +49,22 @@
       {{ nextcloud_container | quote }}
       php occ maintenance:mode --off
 
+    # -------------------------------------------------------------------------
+    # Preflight
+    # -------------------------------------------------------------------------
+
     preflight_script: |
       set -euo pipefail
 
+      echo "Checking Docker"
       command -v docker >/dev/null
       docker version >/dev/null
       docker compose version >/dev/null
 
+      echo "Checking Docker Compose file"
       test -f {{ nextcloud_compose_file | quote }}
 
+      echo "Checking required containers"
       for container in \
         {{ nextcloud_container | quote }} \
         {{ nextcloud_db_container | quote }} \
@@ -68,33 +78,78 @@
         fi
       done
 
+      echo "Checking Nextcloud status"
       {{ docker_cmd }} exec -u www-data \
         {{ nextcloud_container | quote }} \
         php occ status
 
-    backup_script: |
+      echo "Checking MariaDB"
-      set -euo pipefail
+      {{ docker_cmd }} exec \
+        {{ nextcloud_db_container | quote }} \
+        sh -c '
+          mariadb-admin ping -h 127.0.0.1 --silent 2>/dev/null ||
+          mysqladmin ping -h 127.0.0.1 --silent 2>/dev/null
+        '
 
+      echo "Checking Redis"
+      {{ docker_cmd }} exec \
+        {{ redis_container | quote }} \
+        redis-cli -h 127.0.0.1 ping |
+        grep -qx PONG
+
+      echo "Preflight checks completed successfully"
+
+    # -------------------------------------------------------------------------
+    # Backup
+    # -------------------------------------------------------------------------
+
+    backup_script: |
+      set -Eeuo pipefail
+
+      current_step="backup initialization"
+
+      trap '
+        rc=$?
+        echo >&2
+        echo "Backup failed" >&2
+        echo "Step: ${current_step}" >&2
+        echo "Line: ${LINENO}" >&2
+        echo "Command: ${BASH_COMMAND}" >&2
+        echo "Return code: ${rc}" >&2
+        exit "${rc}"
+      ' ERR
+
+      current_step="creating backup directory"
+      echo "Creating backup directory: {{ backup_dir }}"
       mkdir -p {{ backup_dir | quote }}
 
+      current_step="enabling maintenance mode"
       echo "Enabling Nextcloud maintenance mode"
       {{ docker_cmd }} exec -u www-data \
         {{ nextcloud_container | quote }} \
         php occ maintenance:mode --on
 
+      current_step="creating configuration archive"
       echo "Backing up Nextcloud configuration and custom applications"
       {{ docker_cmd }} exec \
         {{ nextcloud_container | quote }} \
-        sh -c 'tar czf /tmp/nextcloud_conf.tgz -C /var/www/html config custom_apps'
+        sh -c '
+          tar czf /tmp/nextcloud_conf.tgz \
+            -C /var/www/html \
+            config custom_apps
+        '
 
+      current_step="copying configuration archive"
       {{ docker_cmd }} cp \
         {{ nextcloud_container | quote }}:/tmp/nextcloud_conf.tgz \
         {{ (backup_dir ~ '/nextcloud_conf.tgz') | quote }}
 
+      current_step="removing temporary configuration archive"
       {{ docker_cmd }} exec \
         {{ nextcloud_container | quote }} \
         rm -f /tmp/nextcloud_conf.tgz
 
+      current_step="creating database dump"
       echo "Backing up Nextcloud database"
       {{ docker_cmd }} exec \
         {{ nextcloud_db_container | quote }} \
@@ -112,20 +167,50 @@
           fi
         ' > {{ (backup_dir ~ '/db.sql') | quote }}
 
+      current_step="validating configuration archive"
       test -s {{ (backup_dir ~ '/nextcloud_conf.tgz') | quote }}
+
+      current_step="validating database dump"
       test -s {{ (backup_dir ~ '/db.sql') | quote }}
 
-      echo "Backup completed successfully: {{ backup_dir }}"
+      echo "Backup completed successfully"
+      echo "Backup directory: {{ backup_dir }}"
+
+    # -------------------------------------------------------------------------
+    # Upgrade
+    # -------------------------------------------------------------------------
 
     upgrade_script: |
-      set -euo pipefail
+      set -Eeuo pipefail
 
+      current_step="upgrade initialization"
+
+      trap '
+        rc=$?
+        echo >&2
+        echo "Nextcloud upgrade failed" >&2
+        echo "Step: ${current_step}" >&2
+        echo "Line: ${LINENO}" >&2
+        echo "Command: ${BASH_COMMAND}" >&2
+        echo "Return code: ${rc}" >&2
+        exit "${rc}"
+      ' ERR
+
+      current_step="docker compose config validation"
+      echo "Validating Docker Compose configuration"
+      {{ docker_cmd }} compose \
+        -p {{ nextcloud_project | quote }} \
+        -f {{ nextcloud_compose_file | quote }} \
+        config --quiet
+
+      current_step="docker compose pull"
       echo "Pulling the current Nextcloud image"
       {{ docker_cmd }} compose \
         -p {{ nextcloud_project | quote }} \
         -f {{ nextcloud_compose_file | quote }} \
         pull {{ nextcloud_service | quote }}
 
+      current_step="docker compose up"
       echo "Recreating the Nextcloud container"
       {{ docker_cmd }} compose \
         -p {{ nextcloud_project | quote }} \
@@ -135,40 +220,118 @@
         --force-recreate \
         {{ nextcloud_service | quote }}
 
+      current_step="waiting for the Nextcloud container"
       echo "Waiting for the Nextcloud container"
-      timeout 180s bash -c '
-        until [ "$(docker inspect --format "{% raw %}{{.State.Running}}{% endraw %}" {{ nextcloud_container | quote }} 2>/dev/null)" = "true" ]; do
-          sleep 3
-        done
-      '
 
+      container_deadline=$((SECONDS + 180))
+
+      while true
+      do
+        container_state="$(docker inspect --format '{% raw %}{{.State.Running}}{% endraw %}' {{ nextcloud_container | quote }} 2>/dev/null || true)"
+
+        if [ "${container_state}" = "true" ]; then
+          break
+        fi
+
+        if [ "${SECONDS}" -ge "${container_deadline}" ]; then
+          echo "Timeout waiting for container {{ nextcloud_container }}" >&2
+          exit 124
+        fi
+
+        sleep 3
+      done
+
+      current_step="waiting for Nextcloud OCC"
       echo "Waiting for Nextcloud OCC"
-      timeout 180s bash -c '
-        until docker exec -u www-data {{ nextcloud_container | quote }} php occ status >/dev/null 2>&1; do
-          sleep 3
-        done
-      '
 
+      occ_deadline=$((SECONDS + 180))
+
+      while true
+      do
+        if docker exec -u www-data \
+          {{ nextcloud_container | quote }} \
+          php occ status >/dev/null 2>&1
+        then
+          break
+        fi
+
+        if [ "${SECONDS}" -ge "${occ_deadline}" ]; then
+          echo "Timeout waiting for Nextcloud OCC" >&2
+          exit 124
+        fi
+
+        sleep 3
+      done
+
+      current_step="Nextcloud database upgrade"
       echo "Running the Nextcloud database upgrade"
       {{ docker_cmd }} exec -u www-data \
         {{ nextcloud_container | quote }} \
         php occ upgrade
 
+      current_step="Nextcloud application update"
       echo "Updating Nextcloud applications"
       {{ docker_cmd }} exec -u www-data \
         {{ nextcloud_container | quote }} \
         php occ app:update --all
 
+      current_step="Nextcloud maintenance repair"
       echo "Running Nextcloud maintenance repair"
       {{ docker_cmd }} exec -u www-data \
         {{ nextcloud_container | quote }} \
         php occ maintenance:repair --include-expensive
 
+      current_step="reading final Nextcloud status"
       echo "Reading Nextcloud status"
       {{ docker_cmd }} exec -u www-data \
         {{ nextcloud_container | quote }} \
         php occ status
 
+      echo "Nextcloud upgrade commands completed successfully"
+
+    # -------------------------------------------------------------------------
+    # Post-upgrade checks
+    # -------------------------------------------------------------------------
+
+    postcheck_script: |
+      set -euo pipefail
+
+      echo "Checking required containers after upgrade"
+
+      for container in \
+        {{ nextcloud_container | quote }} \
+        {{ nextcloud_db_container | quote }} \
+        {{ redis_container | quote }}
+      do
+        state="$(docker inspect --format '{% raw %}{{.State.Running}}{% endraw %}' "${container}" 2>/dev/null || true)"
+
+        if [ "${state}" != "true" ]; then
+          echo "Container ${container} is not running" >&2
+          exit 1
+        fi
+      done
+
+      echo "Checking MariaDB after upgrade"
+      {{ docker_cmd }} exec \
+        {{ nextcloud_db_container | quote }} \
+        sh -c '
+          mariadb-admin ping -h 127.0.0.1 --silent 2>/dev/null ||
+          mysqladmin ping -h 127.0.0.1 --silent 2>/dev/null
+        '
+
+      echo "Checking Redis after upgrade"
+      {{ docker_cmd }} exec \
+        {{ redis_container | quote }} \
+        redis-cli -h 127.0.0.1 ping |
+        grep -qx PONG
+
+      echo "Checking Nextcloud status after upgrade"
+      {{ docker_cmd }} exec -u www-data \
+        {{ nextcloud_container | quote }} \
+        php occ status
+
+      echo "Post-upgrade checks completed successfully"
+
   pre_tasks:
     - name: Validate VM connection variables
       ansible.builtin.assert:
@@ -192,6 +355,10 @@
         state: present
         update_cache: true
 
+    # -------------------------------------------------------------------------
+    # Preflight
+    # -------------------------------------------------------------------------
+
     - name: Nextcloud | Run preflight checks on VM
       ansible.builtin.command:
         argv:
@@ -227,7 +394,7 @@
     - name: Nextcloud | Show preflight result
       ansible.builtin.debug:
         msg: |
-          RC: {{ nc_preflight.rc }}
+          Preflight return code: {{ nc_preflight.rc }}
 
           STDOUT:
           {{ nc_preflight.stdout | default('') | trim }}
@@ -236,6 +403,10 @@
           {{ nc_preflight.stderr | default('') | trim }}
       when: debug_level == 1
 
+    # -------------------------------------------------------------------------
+    # Current version
+    # -------------------------------------------------------------------------
+
     - name: Nextcloud | Read current version before upgrade
       ansible.builtin.command:
         argv:
@@ -250,13 +421,19 @@
           - LogLevel=ERROR
           - -o
           - ConnectTimeout=15
+          - -o
+          - ServerAliveInterval=10
+          - -o
+          - ServerAliveCountMax=3
           - "{{ vm_user }}@{{ vm_ip }}"
           - >-
             {{
-              ('sudo -n bash -lc ' ~ (nextcloud_version_command | quote))
+              ('sudo -n bash -lc ' ~
+               (nextcloud_version_command | quote))
               if vm_use_sudo
               else
-              ('bash -lc ' ~ (nextcloud_version_command | quote))
+              ('bash -lc ' ~
+               (nextcloud_version_command | quote))
             }}
       environment:
         SSHPASS: "{{ vm_password }}"
@@ -275,6 +452,10 @@
             | trim
           }}
 
+    # -------------------------------------------------------------------------
+    # Backup and upgrade
+    # -------------------------------------------------------------------------
+
     - name: Nextcloud | Run backup and upgrade
       block:
         - name: Nextcloud | Create configuration and database backup
@@ -306,13 +487,14 @@
           environment:
             SSHPASS: "{{ vm_password }}"
           register: nc_backup
+          changed_when: true
           no_log: true
 
         - name: Nextcloud | Show backup output
           ansible.builtin.debug:
             msg: |
               Backup directory: {{ backup_dir }}
-              RC: {{ nc_backup.rc }}
+              Backup return code: {{ nc_backup.rc }}
 
               STDOUT:
               {{ nc_backup.stdout | default('') | trim }}
@@ -321,7 +503,7 @@
               {{ nc_backup.stderr | default('') | trim }}
           when: debug_level == 1
 
-                - name: Nextcloud | Pull image and upgrade
+        - name: Nextcloud | Pull image and run upgrade
           ansible.builtin.command:
             argv:
               - sshpass
@@ -382,18 +564,6 @@
               {{ nc_upgrade.stderr | default('') | trim }}
             quiet: true
 
-        - name: Nextcloud | Show upgrade output
-          ansible.builtin.debug:
-            msg: |
-              RC: {{ nc_upgrade.rc }}
-
-              STDOUT:
-              {{ nc_upgrade.stdout | default('') | trim }}
-
-              STDERR:
-              {{ nc_upgrade.stderr | default('') | trim }}
-          when: debug_level == 1
-
       always:
         - name: Nextcloud | Disable maintenance mode
           ansible.builtin.command:
@@ -409,6 +579,10 @@
               - LogLevel=ERROR
               - -o
               - ConnectTimeout=15
+              - -o
+              - ServerAliveInterval=10
+              - -o
+              - ServerAliveCountMax=3
               - "{{ vm_user }}@{{ vm_ip }}"
               - >-
                 {{
@@ -429,14 +603,15 @@
         - name: Nextcloud | Show maintenance mode result
           ansible.builtin.debug:
             msg: |
-              RC: {{ nc_maint_off.rc }}
+              Maintenance return code: {{ nc_maint_off.rc }}
 
               STDOUT:
               {{ nc_maint_off.stdout | default('') | trim }}
 
               STDERR:
               {{ nc_maint_off.stderr | default('') | trim }}
-          when: debug_level == 1
+          when:
+            - debug_level == 1 or nc_maint_off.rc != 0
 
         - name: Nextcloud | Verify maintenance mode was disabled
           ansible.builtin.assert:
@@ -444,12 +619,21 @@
               - nc_maint_off.rc == 0
             fail_msg: |
               Failed to disable Nextcloud maintenance mode.
-              RC: {{ nc_maint_off.rc }}
+
+              Return code:
+              {{ nc_maint_off.rc }}
+
+              STDOUT:
+              {{ nc_maint_off.stdout | default('') | trim }}
 
               STDERR:
               {{ nc_maint_off.stderr | default('') | trim }}
             quiet: true
 
+    # -------------------------------------------------------------------------
+    # Public Nextcloud readiness
+    # -------------------------------------------------------------------------
+
     - name: Nextcloud | Wait for public status.php
       ansible.builtin.uri:
         url: "{{ nextcloud_status_url }}"
@@ -471,7 +655,11 @@
         - not (nc_status.json.needsDbUpgrade | default(true) | bool)
       changed_when: false
 
-    - name: Nextcloud | Check required containers after upgrade
+    # -------------------------------------------------------------------------
+    # Post-upgrade checks
+    # -------------------------------------------------------------------------
+
+    - name: Nextcloud | Run post-upgrade checks on VM
       ansible.builtin.command:
         argv:
           - sshpass
@@ -485,46 +673,40 @@
           - LogLevel=ERROR
           - -o
           - ConnectTimeout=15
+          - -o
+          - ServerAliveInterval=10
+          - -o
+          - ServerAliveCountMax=3
           - "{{ vm_user }}@{{ vm_ip }}"
           - >-
             {{
-              (
+              ('sudo -n bash -lc ' ~ (postcheck_script | quote))
-                'sudo -n bash -lc ' ~
-                (
-                  'for container in ' ~
-                  (nextcloud_container | quote) ~ ' ' ~
-                  (nextcloud_db_container | quote) ~ ' ' ~
-                  (redis_container | quote) ~
-                  '; do state="$(docker inspect --format ' ~
-                  ("'{{.State.Running}}'") ~
-                  ' "${container}" 2>/dev/null)"; ' ~
-                  '[ "${state}" = "true" ] || exit 1; done'
-                  | quote
-                )
-              )
               if vm_use_sudo
               else
-              (
+              ('bash -lc ' ~ (postcheck_script | quote))
-                'bash -lc ' ~
-                (
-                  'for container in ' ~
-                  (nextcloud_container | quote) ~ ' ' ~
-                  (nextcloud_db_container | quote) ~ ' ' ~
-                  (redis_container | quote) ~
-                  '; do state="$(docker inspect --format ' ~
-                  ("'{{.State.Running}}'") ~
-                  ' "${container}" 2>/dev/null)"; ' ~
-                  '[ "${state}" = "true" ] || exit 1; done'
-                  | quote
-                )
-              )
             }}
       environment:
         SSHPASS: "{{ vm_password }}"
-      register: nc_containers_after
+      register: nc_postcheck
       changed_when: false
       no_log: true
 
+    - name: Nextcloud | Show post-upgrade checks
+      ansible.builtin.debug:
+        msg: |
+          Post-upgrade return code: {{ nc_postcheck.rc }}
+
+          STDOUT:
+          {{ nc_postcheck.stdout | default('') | trim }}
+
+          STDERR:
+          {{ nc_postcheck.stderr | default('') | trim }}
+      when: debug_level == 1
+
+    # -------------------------------------------------------------------------
+    # External Collabora
+    # -------------------------------------------------------------------------
+
     - name: Nextcloud | Check external Collabora root endpoint
       ansible.builtin.uri:
         url: "{{ collabora_url }}"
@@ -560,17 +742,25 @@
         ('<wopi-discovery' not in
         (collabora_discovery.content | default('')))
 
+    # -------------------------------------------------------------------------
+    # Final result
+    # -------------------------------------------------------------------------
+
     - name: Nextcloud | Print final status
       ansible.builtin.debug:
         msg: |
           Nextcloud update completed successfully.
 
+          Version before update:
+          {{ nc_version_before.stdout | default('?') | trim }}
+
+          Version after update:
+          {{ nc_status.json.versionstring
+             | default(nc_status.json.version | default('?')) }}
+
           Backup directory:
           {{ backup_dir }}
 
-          Nextcloud version:
-          {{ nc_status.json.version | default('?') }}
-
           Installed:
           {{ nc_status.json.installed | default('?') }}
 
@@ -583,6 +773,8 @@
           Nextcloud container: OK
           MariaDB container: OK
           Redis container: OK
+          MariaDB readiness: OK
+          Redis readiness: OK
           Nextcloud public status endpoint: OK
           Collabora root endpoint: OK
           Collabora discovery endpoint: OK