fix: enhance Nextcloud upgrade playbook with improved runtime checks, maintenance mode handling, and application integrity validation

nextcloud/update_nextcloud_v2.yml

@@ -203,12 +203,27 @@
         fi
       done
 
-      current_step="checking Nextcloud status"
+      current_step="checking Nextcloud runtime"
-      echo "Checking Nextcloud status"
+      echo "Checking Nextcloud runtime"
 
+      {{ docker_cmd }} exec \
+        {{ nextcloud_container | quote }} \
+        sh -c '
+          command -v php >/dev/null
+          test -s /var/www/html/occ
+          test -s /var/www/html/version.php
+          test -s /var/www/html/config/config.php
+        '
+
+      current_step="reading Nextcloud status"
+      echo "Reading Nextcloud status"
+
+      # The status command is informational during preflight. A previous failed
+      # upgrade may legitimately leave maintenance mode enabled or a database
+      # upgrade pending. Such a state must not prevent a recovery run.
       {{ docker_cmd }} exec -u www-data \
         {{ nextcloud_container | quote }} \
-        php occ status
+        php occ status || true
 
       current_step="checking MariaDB"
       echo "Checking MariaDB"
@@ -245,6 +260,25 @@
       set -Eeuo pipefail
 
       current_step="backup initialization"
+      maintenance_enabled_by_script="false"
+
+      cleanup() {
+        rc=$?
+
+        if [ "${rc}" -ne 0 ] && \
+           [ "${maintenance_enabled_by_script}" = "true" ]
+        then
+          echo "Backup failed; attempting to disable maintenance mode" >&2
+
+          {{ docker_cmd }} exec -u www-data \
+            {{ nextcloud_container | quote }} \
+            php occ maintenance:mode --off >/dev/null 2>&1 || true
+        fi
+
+        exit "${rc}"
+      }
+
+      trap cleanup EXIT
 
       trap '
         rc=$?
@@ -268,6 +302,8 @@
         {{ nextcloud_container | quote }} \
         php occ maintenance:mode --on
 
+      maintenance_enabled_by_script="true"
+
       current_step="creating configuration archive"
       echo "Backing up Nextcloud configuration and custom applications"
 
@@ -301,11 +337,19 @@
         sh -c '
           if command -v mariadb-dump >/dev/null 2>&1; then
             exec mariadb-dump \
+              --single-transaction \
+              --quick \
+              --routines \
+              --triggers \
               -u"$MYSQL_USER" \
               -p"$MYSQL_PASSWORD" \
               "$MYSQL_DATABASE"
           else
             exec mysqldump \
+              --single-transaction \
+              --quick \
+              --routines \
+              --triggers \
               -u"$MYSQL_USER" \
               -p"$MYSQL_PASSWORD" \
               "$MYSQL_DATABASE"
@@ -327,6 +371,8 @@
         {{ nextcloud_container | quote }} \
         php occ maintenance:mode --off
 
+      maintenance_enabled_by_script="false"
+
       current_step="verifying maintenance mode after backup"
       echo "Verifying maintenance mode after backup"
 
@@ -413,71 +459,124 @@
 
         if [ "${SECONDS}" -ge "${container_deadline}" ]; then
           echo "Timeout waiting for container {{ nextcloud_container }}" >&2
+
+          {{ docker_cmd }} logs \
+            --tail 100 \
+            {{ nextcloud_container | quote }} >&2 || true
+
           exit 124
         fi
 
         sleep 3
       done
 
-      current_step="waiting for Nextcloud OCC"
+      current_step="waiting for Nextcloud initialization"
-      echo "Waiting for Nextcloud OCC"
+      echo "Waiting for Nextcloud container initialization"
 
-      occ_deadline=$((SECONDS + 180))
+      initialization_deadline=$((SECONDS + 300))
 
       while true
       do
-        if {{ docker_cmd }} exec -u www-data \
+        if {{ docker_cmd }} exec \
           {{ nextcloud_container | quote }} \
-          php occ status >/dev/null 2>&1
+          sh -c '
+            set -eu
+
+            # The official image entrypoint eventually replaces PID 1 with
+            # apache2-foreground. Until that happens, application files may
+            # still be copied from /usr/src/nextcloud to /var/www/html.
+            pid1_command="$(
+              tr "\000" " " < /proc/1/cmdline
+            )"
+
+            case "${pid1_command}" in
+              *apache2-foreground*)
+                ;;
+              *)
+                exit 1
+                ;;
+            esac
+
+            command -v php >/dev/null
+
+            test -s /var/www/html/occ
+            test -s /var/www/html/version.php
+            test -s /var/www/html/config/config.php
+
+            test -d /var/www/html/apps
+            test -s /var/www/html/apps/activity/appinfo/info.xml
+          ' >/dev/null 2>&1
         then
           break
         fi
 
-        if [ "${SECONDS}" -ge "${occ_deadline}" ]; then
+        if [ "${SECONDS}" -ge "${initialization_deadline}" ]; then
-          echo "Timeout waiting for Nextcloud OCC" >&2
+          echo "Timeout waiting for Nextcloud initialization" >&2
+
+          {{ docker_cmd }} logs \
+            --tail 200 \
+            {{ nextcloud_container | quote }} >&2 || true
+
           exit 124
         fi
 
         sleep 3
       done
 
-      current_step="disabling stale maintenance mode"
+      current_step="validating Nextcloud PHP runtime"
-      echo "Ensuring maintenance mode is disabled before OCC upgrade"
+      echo "Validating Nextcloud PHP runtime"
 
-      maintenance_state="$(
+      {{ docker_cmd }} exec \
-        {{ docker_cmd }} exec -u www-data \
         {{ nextcloud_container | quote }} \
-          php occ config:system:get maintenance \
+        php -r '
-          2>/dev/null || true
+          require "/var/www/html/version.php";
-      )"
 
-      if [ "${maintenance_state}" = "true" ] || \
+          if (empty($OC_Version)) {
-         [ "${maintenance_state}" = "1" ]; then
+              fwrite(STDERR, "Nextcloud version is unavailable\n");
-        echo "Maintenance mode is enabled; disabling it"
+              exit(1);
+          }
 
-        {{ docker_cmd }} exec -u www-data \
+          echo "Nextcloud code version: ";
+          echo implode(".", $OC_Version), PHP_EOL;
+        '
+
+      current_step="validating bundled application files"
+      echo "Validating bundled Nextcloud application files"
+
+      {{ docker_cmd }} exec \
         {{ nextcloud_container | quote }} \
-          php occ maintenance:mode --off
+        php -r '
-      fi
+          $infoFile = "/var/www/html/apps/activity/appinfo/info.xml";
 
-      current_step="verifying maintenance mode before OCC upgrade"
+          if (!is_file($infoFile) || filesize($infoFile) === 0) {
+              fwrite(STDERR, "Activity application metadata is missing\n");
+              exit(1);
+          }
 
-      maintenance_state="$(
+          $xml = @simplexml_load_file($infoFile);
-        {{ docker_cmd }} exec -u www-data \
-          {{ nextcloud_container | quote }} \
-          php occ config:system:get maintenance \
-          2>/dev/null || true
-      )"
 
-      if [ "${maintenance_state}" = "true" ] || \
+          if ($xml === false) {
-         [ "${maintenance_state}" = "1" ]; then
+              fwrite(STDERR, "Activity application metadata is invalid\n");
-        echo "Maintenance mode could not be disabled" >&2
+              exit(1);
-        exit 1
+          }
-      fi
+
+          if ((string) $xml->id !== "activity") {
+              fwrite(STDERR, "Unexpected activity application ID\n");
+              exit(1);
+          }
+
+          echo "Activity application version: ";
+          echo (string) $xml->version, PHP_EOL;
+        '
+
+      echo "Nextcloud container initialization completed"
 
       current_step="running Nextcloud database upgrade"
       echo "Running the Nextcloud database upgrade"
 
+      # Do not disable maintenance mode before this command. A previous failed
+      # upgrade may have intentionally left maintenance mode enabled. OCC
+      # upgrade manages the required maintenance state itself.
       {{ docker_cmd }} exec -u www-data \
         {{ nextcloud_container | quote }} \
         php occ upgrade --no-interaction
@@ -503,6 +602,42 @@
         {{ nextcloud_container | quote }} \
         php occ status
 
+      current_step="verifying final Nextcloud state"
+      echo "Verifying final Nextcloud state"
+
+      maintenance_state="$(
+        {{ docker_cmd }} exec -u www-data \
+          {{ nextcloud_container | quote }} \
+          php occ config:system:get maintenance \
+          2>/dev/null || true
+      )"
+
+      if [ "${maintenance_state}" = "true" ] || \
+         [ "${maintenance_state}" = "1" ]; then
+        echo "Nextcloud is still in maintenance mode" >&2
+        exit 1
+      fi
+
+      needs_db_upgrade="$(
+        {{ docker_cmd }} exec -u www-data \
+          {{ nextcloud_container | quote }} \
+          php occ status --output=json |
+          php -r '
+            $status = json_decode(stream_get_contents(STDIN), true);
+
+            if (!is_array($status)) {
+                exit(2);
+            }
+
+            echo !empty($status["needsDbUpgrade"]) ? "true" : "false";
+          '
+      )"
+
+      if [ "${needs_db_upgrade}" != "false" ]; then
+        echo "Nextcloud still requires a database upgrade" >&2
+        exit 1
+      fi
+
       echo "Nextcloud upgrade commands completed successfully"
 
     # -------------------------------------------------------------------------
@@ -593,6 +728,35 @@
         exit 1
       fi
 
+      current_step="checking database upgrade state"
+
+      needs_db_upgrade="$(
+        {{ docker_cmd }} exec -u www-data \
+          {{ nextcloud_container | quote }} \
+          php occ status --output=json |
+          php -r '
+            $status = json_decode(stream_get_contents(STDIN), true);
+
+            if (!is_array($status)) {
+                exit(2);
+            }
+
+            echo !empty($status["needsDbUpgrade"]) ? "true" : "false";
+          '
+      )"
+
+      if [ "${needs_db_upgrade}" != "false" ]; then
+        echo "Nextcloud still requires a database upgrade" >&2
+        exit 1
+      fi
+
+      current_step="checking activity application integrity"
+      echo "Checking activity application integrity"
+
+      {{ docker_cmd }} exec -u www-data \
+        {{ nextcloud_container | quote }} \
+        php occ integrity:check-app activity
+
       echo "Post-upgrade checks completed successfully"
 
   pre_tasks:
@@ -660,7 +824,6 @@
             src: "{{ nextcloud_compose_local_file }}"
             dest: "{{ nextcloud_compose_controller_staging_file }}"
             mode: "0600"
-          register: nextcloud_compose_controller_copy
 
         - name: Nextcloud | Upload Compose file to VM staging path
           ansible.builtin.command:
@@ -681,8 +844,8 @@
           environment:
             SSHPASS: "{{ vm_password }}"
           register: nextcloud_compose_upload
-          changed_when: nextcloud_compose_upload.rc == 0
+          changed_when: false
-          no_log: false
+          no_log: true
 
         - name: Nextcloud | Validate and install Compose file on VM
           ansible.builtin.command:
@@ -1035,6 +1198,10 @@
             fail_msg: |
               Nextcloud upgrade failed.
 
+              Maintenance mode has intentionally not been disabled because
+              the upgrade command failed. Resolve the reported error and rerun
+              the playbook or complete the OCC upgrade manually.
+
               Return code:
               {{ nc_upgrade.rc }}
 
@@ -1046,6 +1213,11 @@
             quiet: true
 
       always:
+        # Disable maintenance mode when:
+        # - the backup failed before the upgrade task was started, or
+        # - the upgrade completed successfully.
+        #
+        # Do not disable maintenance mode after a failed OCC upgrade.
         - name: Nextcloud | Disable maintenance mode
           ansible.builtin.command:
             argv:
@@ -1080,6 +1252,8 @@
           changed_when: false
           failed_when: false
           no_log: true
+          when:
+            - nc_upgrade is not defined or nc_upgrade.rc == 0
 
         - name: Nextcloud | Show maintenance mode result
           ansible.builtin.debug:
@@ -1092,6 +1266,7 @@
               STDERR:
               {{ nc_maint_off.stderr | default('') | trim }}
           when:
+            - nc_upgrade is not defined or nc_upgrade.rc == 0
             - debug_level == 1 or nc_maint_off.rc != 0
 
         - name: Nextcloud | Verify maintenance mode was disabled
@@ -1110,6 +1285,8 @@
               STDERR:
               {{ nc_maint_off.stderr | default('') | trim }}
             quiet: true
+          when:
+            - nc_upgrade is not defined or nc_upgrade.rc == 0
 
     # -------------------------------------------------------------------------
     # Public Nextcloud readiness
@@ -1283,6 +1460,7 @@
           Redis container: OK
           MariaDB readiness: OK
           Redis readiness: OK
+          Nextcloud application integrity: OK
           Nextcloud public status endpoint: OK
           Collabora root endpoint: OK
           Collabora discovery endpoint: OK