fix: enhance Nextcloud update playbook with improved backup and upgrade logic, and add container readiness checks

nextcloud/update_nextcloud_v2.yml

@@ -8,160 +8,165 @@
   become_method: sudo
 
   vars:
-    # Normalized values from Semaphore Variable Group
+    # vm_pass is stored in Semaphore Secrets as an environment variable.
+    vm_password: "{{ lookup('ansible.builtin.env', 'vm_pass') }}"
+
+    # Values from Semaphore Extra variables.
     vm_use_sudo: "{{ use_sudo | default(false) | bool }}"
     debug_level: "{{ DEBUG | default(0) | int }}"
     retry_count: "{{ RETRIES | default(25) | int }}"
 
-    # Nextcloud configuration
+    # Nextcloud configuration.
     nextcloud_project: "nextcloud-collabora"
     nextcloud_compose_file: "/data/compose/nextcloud/docker-compose-nextcloud.yml"
     nextcloud_service: "nextcloud"
 
     nextcloud_container: "nextcloud"
     nextcloud_db_container: "nextcloud-db"
+    redis_container: "redis"
 
     nextcloud_base_url: "https://cloud.martinfencl.eu"
     nextcloud_status_url: "{{ nextcloud_base_url }}/status.php"
 
+    collabora_url: "https://collabora.martinfencl.eu/"
+    collabora_discovery_url: "https://collabora.martinfencl.eu/hosting/discovery"
+
     backup_timestamp: "{{ lookup('pipe', 'date +%F-%H%M%S') }}"
     backup_dir: "/data/compose/nextcloud/backup-{{ backup_timestamp }}"
 
-    docker_prefix: >-
+    docker_cmd: "env DOCKER_CLI_HINTS=0 docker"
-      unalias docker 2>/dev/null || true;
-      export DOCKER_CLI_HINTS=0;
-      command docker
 
     nextcloud_version_command: >-
-      docker exec -u www-data
+      {{ docker_cmd }} exec -u www-data
-      {{ nextcloud_container }}
+      {{ nextcloud_container | quote }}
       php occ -V
 
     nextcloud_maintenance_off_command: >-
-      docker exec -u www-data
+      {{ docker_cmd }} exec -u www-data
-      {{ nextcloud_container }}
+      {{ nextcloud_container | quote }}
       php occ maintenance:mode --off
 
-    # Commands executed before the container update
+    preflight_script: |
-    nextcloud_backup_commands:
+      set -euo pipefail
-      - >-
-        mkdir -p {{ backup_dir | quote }}
 
-      - >-
+      command -v docker >/dev/null
-        {{ docker_prefix }}
+      docker version >/dev/null
-        exec -u www-data
+      docker compose version >/dev/null
-        {{ nextcloud_container }}
+
+      test -f {{ nextcloud_compose_file | quote }}
+
+      for container in \
+        {{ nextcloud_container | quote }} \
+        {{ nextcloud_db_container | quote }} \
+        {{ redis_container | quote }}
+      do
+        state="$(docker inspect --format '{% raw %}{{.State.Running}}{% endraw %}' "${container}" 2>/dev/null || true)"
+
+        if [ "${state}" != "true" ]; then
+          echo "Container ${container} is not running" >&2
+          exit 1
+        fi
+      done
+
+      {{ docker_cmd }} exec -u www-data \
+        {{ nextcloud_container | quote }} \
+        php occ status
+
+    backup_script: |
+      set -euo pipefail
+
+      mkdir -p {{ backup_dir | quote }}
+
+      echo "Enabling Nextcloud maintenance mode"
+      {{ docker_cmd }} exec -u www-data \
+        {{ nextcloud_container | quote }} \
         php occ maintenance:mode --on
 
-      - >-
+      echo "Backing up Nextcloud configuration and custom applications"
-        {{ docker_prefix }}
+      {{ docker_cmd }} exec \
-        exec
+        {{ nextcloud_container | quote }} \
-        {{ nextcloud_container }}
+        sh -c 'tar czf /tmp/nextcloud_conf.tgz -C /var/www/html config custom_apps'
-        sh -c
-        'tar czf /tmp/nextcloud_conf.tgz
-        -C /var/www/html
-        config custom_apps'
 
-      - >-
+      {{ docker_cmd }} cp \
-        {{ docker_prefix }}
+        {{ nextcloud_container | quote }}:/tmp/nextcloud_conf.tgz \
-        cp
-        {{ nextcloud_container }}:/tmp/nextcloud_conf.tgz
         {{ (backup_dir ~ '/nextcloud_conf.tgz') | quote }}
 
-      - >-
+      {{ docker_cmd }} exec \
-        {{ docker_prefix }}
+        {{ nextcloud_container | quote }} \
-        exec
-        {{ nextcloud_container }}
         rm -f /tmp/nextcloud_conf.tgz
 
-      - >-
+      echo "Backing up Nextcloud database"
-        set -o pipefail;
+      {{ docker_cmd }} exec \
-        {{ docker_prefix }}
+        {{ nextcloud_db_container | quote }} \
-        exec
+        sh -c '
-        {{ nextcloud_db_container }}
+          if command -v mariadb-dump >/dev/null 2>&1; then
-        sh -c
+            exec mariadb-dump \
-        'if command -v mariadb-dump >/dev/null 2>&1; then
+              -u"$MYSQL_USER" \
-           exec mariadb-dump
+              -p"$MYSQL_PASSWORD" \
-             -u"$MYSQL_USER"
+              "$MYSQL_DATABASE"
-             -p"$MYSQL_PASSWORD"
+          else
-             "$MYSQL_DATABASE";
+            exec mysqldump \
-         else
+              -u"$MYSQL_USER" \
-           exec mysqldump
+              -p"$MYSQL_PASSWORD" \
-             -u"$MYSQL_USER"
+              "$MYSQL_DATABASE"
-             -p"$MYSQL_PASSWORD"
+          fi
-             "$MYSQL_DATABASE";
+        ' > {{ (backup_dir ~ '/db.sql') | quote }}
-         fi'
-        > {{ (backup_dir ~ '/db.sql') | quote }}
 
-      - >-
+      test -s {{ (backup_dir ~ '/nextcloud_conf.tgz') | quote }}
-        test -s
+      test -s {{ (backup_dir ~ '/db.sql') | quote }}
-        {{ (backup_dir ~ '/nextcloud_conf.tgz') | quote }}
 
-      - >-
+      echo "Backup completed successfully: {{ backup_dir }}"
-        test -s
-        {{ (backup_dir ~ '/db.sql') | quote }}
 
-    # Commands executed while maintenance mode is enabled
+    upgrade_script: |
-    nextcloud_upgrade_commands:
+      set -euo pipefail
-      - >-
+
-        {{ docker_prefix }}
+      echo "Pulling the current Nextcloud image"
-        compose
+      {{ docker_cmd }} compose \
-        -p {{ nextcloud_project | quote }}
+        -p {{ nextcloud_project | quote }} \
-        -f {{ nextcloud_compose_file | quote }}
+        -f {{ nextcloud_compose_file | quote }} \
-        pull
+        pull {{ nextcloud_service | quote }}
+
+      echo "Recreating the Nextcloud container"
+      {{ docker_cmd }} compose \
+        -p {{ nextcloud_project | quote }} \
+        -f {{ nextcloud_compose_file | quote }} \
+        up -d \
+        --no-deps \
+        --force-recreate \
         {{ nextcloud_service | quote }}
 
-      - >-
+      echo "Waiting for the Nextcloud container"
-        {{ docker_prefix }}
+      timeout 180s bash -c '
-        compose
+        until [ "$(docker inspect --format "{% raw %}{{.State.Running}}{% endraw %}" {{ nextcloud_container | quote }} 2>/dev/null)" = "true" ]; do
-        -p {{ nextcloud_project | quote }}
+          sleep 3
-        -f {{ nextcloud_compose_file | quote }}
+        done
-        up -d
+      '
-        --no-deps
-        --force-recreate
-        {{ nextcloud_service | quote }}
 
-      - >-
+      echo "Waiting for Nextcloud OCC"
-        timeout 180s bash -c
+      timeout 180s bash -c '
-        'until docker inspect
+        until docker exec -u www-data {{ nextcloud_container | quote }} php occ status >/dev/null 2>&1; do
-        --format="{{ '{{' }}.State.Running{{ '}}' }}"
+          sleep 3
-        {{ nextcloud_container | quote }}
+        done
-        2>/dev/null | grep -qx true;
+      '
-        do
-          sleep 3;
-        done'
 
-      - >-
+      echo "Running the Nextcloud database upgrade"
-        timeout 180s bash -c
+      {{ docker_cmd }} exec -u www-data \
-        'until docker exec -u www-data
+        {{ nextcloud_container | quote }} \
-        {{ nextcloud_container | quote }}
-        php occ status >/dev/null 2>&1;
-        do
-          sleep 3;
-        done'
-
-      - >-
-        {{ docker_prefix }}
-        exec -u www-data
-        {{ nextcloud_container }}
         php occ upgrade
 
-      - >-
+      echo "Updating Nextcloud applications"
-        {{ docker_prefix }}
+      {{ docker_cmd }} exec -u www-data \
-        exec -u www-data
+        {{ nextcloud_container | quote }} \
-        {{ nextcloud_container }}
         php occ app:update --all
 
-      - >-
+      echo "Running Nextcloud maintenance repair"
-        {{ docker_prefix }}
+      {{ docker_cmd }} exec -u www-data \
-        exec -u www-data
+        {{ nextcloud_container | quote }} \
-        {{ nextcloud_container }}
         php occ maintenance:repair --include-expensive
 
-      - >-
+      echo "Reading Nextcloud status"
-        {{ docker_prefix }}
+      {{ docker_cmd }} exec -u www-data \
-        exec -u www-data
+        {{ nextcloud_container | quote }} \
-        {{ nextcloud_container }}
         php occ status
 
   pre_tasks:
@@ -172,12 +177,11 @@
           - vm_ip | string | trim | length > 0
           - vm_user is defined
           - vm_user | string | trim | length > 0
-          - vm_pass is defined
+          - vm_password | string | length > 0
-          - vm_pass | string | length > 0
           - retry_count | int > 0
         fail_msg: >-
           Missing or invalid vm_ip, vm_user, vm_pass or RETRIES.
-          Configure them in the attached Semaphore Variable Group.
+          Check the attached Semaphore Variable Group.
         quiet: true
       no_log: true
 
@@ -188,7 +192,7 @@
         state: present
         update_cache: true
 
-    - name: Nextcloud | Show current version before upgrade
+    - name: Nextcloud | Run preflight checks on VM
       ansible.builtin.command:
         argv:
           - sshpass
@@ -199,41 +203,81 @@
           - -o
           - UserKnownHostsFile=/dev/null
           - -o
+          - LogLevel=ERROR
+          - -o
+          - ConnectTimeout=15
+          - -o
+          - ServerAliveInterval=10
+          - -o
+          - ServerAliveCountMax=3
+          - "{{ vm_user }}@{{ vm_ip }}"
+          - >-
+            {{
+              ('sudo -n bash -lc ' ~ (preflight_script | quote))
+              if vm_use_sudo
+              else
+              ('bash -lc ' ~ (preflight_script | quote))
+            }}
+      environment:
+        SSHPASS: "{{ vm_password }}"
+      register: nc_preflight
+      changed_when: false
+      no_log: true
+
+    - name: Nextcloud | Show preflight result
+      ansible.builtin.debug:
+        msg: |
+          RC: {{ nc_preflight.rc }}
+
+          STDOUT:
+          {{ nc_preflight.stdout | default('') | trim }}
+
+          STDERR:
+          {{ nc_preflight.stderr | default('') | trim }}
+      when: debug_level == 1
+
+    - name: Nextcloud | Read current version before upgrade
+      ansible.builtin.command:
+        argv:
+          - sshpass
+          - -e
+          - ssh
+          - -o
+          - StrictHostKeyChecking=no
+          - -o
+          - UserKnownHostsFile=/dev/null
+          - -o
+          - LogLevel=ERROR
+          - -o
           - ConnectTimeout=15
           - "{{ vm_user }}@{{ vm_ip }}"
           - >-
             {{
-              ('sudo -n bash -lc ' ~
+              ('sudo -n bash -lc ' ~ (nextcloud_version_command | quote))
-               (nextcloud_version_command | quote))
               if vm_use_sudo
               else
-              ('bash -lc ' ~
+              ('bash -lc ' ~ (nextcloud_version_command | quote))
-               (nextcloud_version_command | quote))
             }}
       environment:
-        SSHPASS: "{{ vm_pass }}"
+        SSHPASS: "{{ vm_password }}"
       register: nc_version_before
       changed_when: false
       failed_when: false
-      when: debug_level == 1
+      no_log: true
 
     - name: Nextcloud | Print current version
       ansible.builtin.debug:
         msg: >-
+          Current version:
           {{
             nc_version_before.stdout
             | default('Nextcloud version is unavailable')
             | trim
           }}
-      when: debug_level == 1
 
     - name: Nextcloud | Run backup and upgrade
       block:
-        # ---------------------------------------------------------------------
+        - name: Nextcloud | Create configuration and database backup
-        # Backup
-        # ---------------------------------------------------------------------
-
-        - name: Nextcloud | Run backup commands on VM
           ansible.builtin.command:
             argv:
               - sshpass
@@ -244,6 +288,8 @@
               - -o
               - UserKnownHostsFile=/dev/null
               - -o
+              - LogLevel=ERROR
+              - -o
               - ConnectTimeout=15
               - -o
               - ServerAliveInterval=10
@@ -252,60 +298,30 @@
               - "{{ vm_user }}@{{ vm_ip }}"
               - >-
                 {{
-                  ('sudo -n bash -lc ' ~ (item | quote))
+                  ('sudo -n bash -lc ' ~ (backup_script | quote))
                   if vm_use_sudo
                   else
-                  ('bash -lc ' ~ (item | quote))
+                  ('bash -lc ' ~ (backup_script | quote))
                 }}
           environment:
-            SSHPASS: "{{ vm_pass }}"
+            SSHPASS: "{{ vm_password }}"
-          loop: "{{ nextcloud_backup_commands }}"
+          register: nc_backup
-          loop_control:
+          no_log: true
-            index_var: backup_index
-            label: "backup-cmd-{{ backup_index }}"
-          register: nc_backup_cmds
-          changed_when: false
-          failed_when: false
 
-        - name: Nextcloud | Show backup command outputs
+        - name: Nextcloud | Show backup output
           ansible.builtin.debug:
             msg: |
-              CMD: {{ item.item }}
+              Backup directory: {{ backup_dir }}
-              RC:  {{ item.rc }}
+              RC: {{ nc_backup.rc }}
 
               STDOUT:
-              {{ item.stdout | default('') | trim }}
+              {{ nc_backup.stdout | default('') | trim }}
 
               STDERR:
-              {{ item.stderr | default('') | trim }}
+              {{ nc_backup.stderr | default('') | trim }}
-          loop: "{{ nc_backup_cmds.results }}"
-          loop_control:
-            index_var: backup_debug_index
-            label: "backup-cmd-{{ backup_debug_index }}"
           when: debug_level == 1
 
-        - name: Nextcloud | Fail if any backup command failed
+        - name: Nextcloud | Pull image and upgrade
-          ansible.builtin.assert:
-            that:
-              - item.rc == 0
-            fail_msg: |
-              Nextcloud backup command failed.
-              Command: {{ item.item }}
-              RC: {{ item.rc }}
-
-              STDERR:
-              {{ item.stderr | default('') | trim }}
-            quiet: true
-          loop: "{{ nc_backup_cmds.results }}"
-          loop_control:
-            index_var: backup_assert_index
-            label: "backup-cmd-{{ backup_assert_index }}"
-
-        # ---------------------------------------------------------------------
-        # Upgrade
-        # ---------------------------------------------------------------------
-
-        - name: Nextcloud | Run upgrade commands on VM
           ansible.builtin.command:
             argv:
               - sshpass
@@ -316,6 +332,8 @@
               - -o
               - UserKnownHostsFile=/dev/null
               - -o
+              - LogLevel=ERROR
+              - -o
               - ConnectTimeout=15
               - -o
               - ServerAliveInterval=10
@@ -324,55 +342,28 @@
               - "{{ vm_user }}@{{ vm_ip }}"
               - >-
                 {{
-                  ('sudo -n bash -lc ' ~ (item | quote))
+                  ('sudo -n bash -lc ' ~ (upgrade_script | quote))
                   if vm_use_sudo
                   else
-                  ('bash -lc ' ~ (item | quote))
+                  ('bash -lc ' ~ (upgrade_script | quote))
                 }}
           environment:
-            SSHPASS: "{{ vm_pass }}"
+            SSHPASS: "{{ vm_password }}"
-          loop: "{{ nextcloud_upgrade_commands }}"
+          register: nc_upgrade
-          loop_control:
+          no_log: true
-            index_var: upgrade_index
-            label: "upgrade-cmd-{{ upgrade_index }}"
-          register: nc_upgrade_cmds
-          changed_when: false
-          failed_when: false
 
-        - name: Nextcloud | Show upgrade command outputs
+        - name: Nextcloud | Show upgrade output
           ansible.builtin.debug:
             msg: |
-              CMD: {{ item.item }}
+              RC: {{ nc_upgrade.rc }}
-              RC:  {{ item.rc }}
 
               STDOUT:
-              {{ item.stdout | default('') | trim }}
+              {{ nc_upgrade.stdout | default('') | trim }}
 
               STDERR:
-              {{ item.stderr | default('') | trim }}
+              {{ nc_upgrade.stderr | default('') | trim }}
-          loop: "{{ nc_upgrade_cmds.results }}"
-          loop_control:
-            index_var: upgrade_debug_index
-            label: "upgrade-cmd-{{ upgrade_debug_index }}"
           when: debug_level == 1
 
-        - name: Nextcloud | Fail if any upgrade command failed
-          ansible.builtin.assert:
-            that:
-              - item.rc == 0
-            fail_msg: |
-              Nextcloud upgrade command failed.
-              Command: {{ item.item }}
-              RC: {{ item.rc }}
-
-              STDERR:
-              {{ item.stderr | default('') | trim }}
-            quiet: true
-          loop: "{{ nc_upgrade_cmds.results }}"
-          loop_control:
-            index_var: upgrade_assert_index
-            label: "upgrade-cmd-{{ upgrade_assert_index }}"
-
       always:
         - name: Nextcloud | Disable maintenance mode
           ansible.builtin.command:
@@ -385,6 +376,8 @@
               - -o
               - UserKnownHostsFile=/dev/null
               - -o
+              - LogLevel=ERROR
+              - -o
               - ConnectTimeout=15
               - "{{ vm_user }}@{{ vm_ip }}"
               - >-
@@ -397,10 +390,11 @@
                    (nextcloud_maintenance_off_command | quote))
                 }}
           environment:
-            SSHPASS: "{{ vm_pass }}"
+            SSHPASS: "{{ vm_password }}"
           register: nc_maint_off
           changed_when: false
           failed_when: false
+          no_log: true
 
         - name: Nextcloud | Show maintenance mode result
           ansible.builtin.debug:
@@ -414,7 +408,7 @@
               {{ nc_maint_off.stderr | default('') | trim }}
           when: debug_level == 1
 
-        - name: Nextcloud | Fail if maintenance mode could not be disabled
+        - name: Nextcloud | Verify maintenance mode was disabled
           ansible.builtin.assert:
             that:
               - nc_maint_off.rc == 0
@@ -426,36 +420,140 @@
               {{ nc_maint_off.stderr | default('') | trim }}
             quiet: true
 
-    # -------------------------------------------------------------------------
+    - name: Nextcloud | Wait for public status.php
-    # Readiness check
-    # -------------------------------------------------------------------------
-
-    - name: Nextcloud | Wait for status.php
       ansible.builtin.uri:
         url: "{{ nextcloud_status_url }}"
         method: GET
         return_content: true
         validate_certs: true
         status_code: 200
+        timeout: 20
       register: nc_status
       delegate_to: localhost
       run_once: true
       retries: "{{ retry_count }}"
       delay: 4
       until:
-        - nc_status.status | default(0) == 200
+        - nc_status.status | default(0) | int == 200
         - nc_status.json is defined
         - nc_status.json.installed | default(false) | bool
         - not (nc_status.json.maintenance | default(true) | bool)
         - not (nc_status.json.needsDbUpgrade | default(true) | bool)
       changed_when: false
 
+    - name: Nextcloud | Check required containers after upgrade
+      ansible.builtin.command:
+        argv:
+          - sshpass
+          - -e
+          - ssh
+          - -o
+          - StrictHostKeyChecking=no
+          - -o
+          - UserKnownHostsFile=/dev/null
+          - -o
+          - LogLevel=ERROR
+          - -o
+          - ConnectTimeout=15
+          - "{{ vm_user }}@{{ vm_ip }}"
+          - >-
+            {{
+              (
+                'sudo -n bash -lc ' ~
+                (
+                  'for container in ' ~
+                  (nextcloud_container | quote) ~ ' ' ~
+                  (nextcloud_db_container | quote) ~ ' ' ~
+                  (redis_container | quote) ~
+                  '; do state="$(docker inspect --format ' ~
+                  ("'{{.State.Running}}'") ~
+                  ' "${container}" 2>/dev/null)"; ' ~
+                  '[ "${state}" = "true" ] || exit 1; done'
+                  | quote
+                )
+              )
+              if vm_use_sudo
+              else
+              (
+                'bash -lc ' ~
+                (
+                  'for container in ' ~
+                  (nextcloud_container | quote) ~ ' ' ~
+                  (nextcloud_db_container | quote) ~ ' ' ~
+                  (redis_container | quote) ~
+                  '; do state="$(docker inspect --format ' ~
+                  ("'{{.State.Running}}'") ~
+                  ' "${container}" 2>/dev/null)"; ' ~
+                  '[ "${state}" = "true" ] || exit 1; done'
+                  | quote
+                )
+              )
+            }}
+      environment:
+        SSHPASS: "{{ vm_password }}"
+      register: nc_containers_after
+      changed_when: false
+      no_log: true
+
+    - name: Nextcloud | Check external Collabora root endpoint
+      ansible.builtin.uri:
+        url: "{{ collabora_url }}"
+        method: GET
+        return_content: true
+        validate_certs: true
+        status_code: 200
+        timeout: 20
+      register: collabora_root
+      delegate_to: localhost
+      run_once: true
+      changed_when: false
+      failed_when: >-
+        (collabora_root.status | default(0) | int != 200)
+        or
+        ('OK' not in (collabora_root.content | default('')))
+
+    - name: Nextcloud | Check external Collabora discovery endpoint
+      ansible.builtin.uri:
+        url: "{{ collabora_discovery_url }}"
+        method: GET
+        return_content: true
+        validate_certs: true
+        status_code: 200
+        timeout: 20
+      register: collabora_discovery
+      delegate_to: localhost
+      run_once: true
+      changed_when: false
+      failed_when: >-
+        (collabora_discovery.status | default(0) | int != 200)
+        or
+        ('<wopi-discovery' not in
+        (collabora_discovery.content | default('')))
+
     - name: Nextcloud | Print final status
       ansible.builtin.debug:
-        msg: >-
+        msg: |
-          Nextcloud {{ nc_status.json.version | default('?') }}
+          Nextcloud update completed successfully.
-          (installed={{ nc_status.json.installed | default('?') }},
+
-          maintenance={{ nc_status.json.maintenance | default('?') }},
+          Backup directory:
-          needsDbUpgrade={{
+          {{ backup_dir }}
-            nc_status.json.needsDbUpgrade | default('?')
+
-          }})
+          Nextcloud version:
+          {{ nc_status.json.version | default('?') }}
+
+          Installed:
+          {{ nc_status.json.installed | default('?') }}
+
+          Maintenance:
+          {{ nc_status.json.maintenance | default('?') }}
+
+          Needs database upgrade:
+          {{ nc_status.json.needsDbUpgrade | default('?') }}
+
+          Nextcloud container: OK
+          MariaDB container: OK
+          Redis container: OK
+          Nextcloud public status endpoint: OK
+          Collabora root endpoint: OK
+          Collabora discovery endpoint: OK
+      run_once: true