Add replica rebuild wait;Upgrade longhorn&harbor

ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/containerimages.yml

@@ -0,0 +1,50 @@
+- block:
+
+    - name: Push images to registry
+      ansible.builtin.shell:
+        cmd: >-
+          skopeo copy \
+            --insecure-policy \
+            --dest-tls-verify=false \
+            --dest-creds admin:{{ vapp['metacluster.password'] }} \
+            docker-archive:./{{ item | basename }} \
+            docker://registry.{{ vapp['metacluster.fqdn'] }}/library/$( \
+              skopeo list-tags \
+                --insecure-policy \
+                docker-archive:./{{ item | basename }} | \
+              jq -r '.Tags[0]')
+        chdir: /opt/metacluster/container-images/
+      register: push_result
+      loop: "{{ query('ansible.builtin.fileglob', '/opt/metacluster/container-images/*.tar') | sort }}"
+      loop_control:
+        label: "{{ item | basename }}"
+      retries: "{{ playbook.retries }}"
+      delay: "{{ playbook.delays.short }}"
+      until: push_result is not failed
+
+    - name: Get all stored container images (=artifacts)
+      ansible.builtin.uri:
+        url: https://registry.{{ vapp['metacluster.fqdn'] }}/api/v2.0/search?q=library
+        method: GET
+      register: registry_artifacts
+
+    - name: Get source registries of all artifacts
+      ansible.builtin.set_fact:
+        source_registries: "{{ (source_registries | default([]) + [(item | split('/'))[1]]) | unique | sort }}"
+      loop: "{{ registry_artifacts.json.repository | json_query('[*].repository_name') }}"
+
+    - name: Configure K3s node for private registry
+      ansible.builtin.template:
+        dest: /etc/rancher/k3s/registries.yaml
+        src: registries.j2
+      vars:
+        _template:
+          data: "{{ source_registries }}"
+          hv:
+            fqdn: "{{ vapp['metacluster.fqdn'] }}"
+
+  module_defaults:
+    ansible.builtin.uri:
+      validate_certs: no
+      status_code: [200, 201, 401]
+      body_format: json

ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/main.yml

@@ -1,9 +1,11 @@
 - import_tasks: init.yml
-- import_tasks: registry.yml
+- import_tasks: containerimages.yml
 - import_tasks: k3s.yml
 - import_tasks: assets.yml
-# - import_tasks: ingress.yml
 - import_tasks: storage.yml
+
+# - import_tasks: charts.yml
+- import_tasks: registry.yml
 # - import_tasks: certauthority.yml
 # - import_tasks: git.yml
 # - import_tasks: gitops.yml

ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/registry.yml

@@ -1,47 +1,25 @@
 - block:
 
-    - name: Push images to registry
-      ansible.builtin.shell:
-        cmd: >-
-          skopeo copy \
-            --insecure-policy \
-            --dest-tls-verify=false \
-            --dest-creds admin:{{ vapp['metacluster.password'] }} \
-            docker-archive:./{{ item | basename }} \
-            docker://registry.{{ vapp['metacluster.fqdn'] }}/library/$( \
-              skopeo list-tags \
-                --insecure-policy \
-                docker-archive:./{{ item | basename }} | \
-              jq -r '.Tags[0]')
-        chdir: /opt/metacluster/container-images/
-      register: push_result
-      loop: "{{ query('ansible.builtin.fileglob', '/opt/metacluster/container-images/*.tar') | sort }}"
-      loop_control:
-        label: "{{ item | basename }}"
-      retries: "{{ playbook.retries }}"
-      delay: "{{ playbook.delays.short }}"
-      until: push_result is not failed
+    - name: Upgrade harbor chart
+      kubernetes.core.helm:
+        name: harbor
+        chart_ref: /opt/metacluster/helm-charts/harbor
+        release_namespace: harbor
+        create_namespace: yes
+        wait: no
+        kubeconfig: "{{ kubeconfig.path }}"
+        values: "{{ components.harbor.chart_values }}"
 
-    - name: Get all stored container images (=artifacts)
+    - name: Ensure harbor API availability
       ansible.builtin.uri:
-        url: https://registry.{{ vapp['metacluster.fqdn'] }}/api/v2.0/search?q=library
+        url: https://registry.{{ vapp['metacluster.fqdn'] }}/api/v2.0/health
         method: GET
-      register: registry_artifacts
-
-    - name: Get source registries of all artifacts
-      ansible.builtin.set_fact:
-        source_registries: "{{ (source_registries | default([]) + [(item | split('/'))[1]]) | unique | sort }}"
-      loop: "{{ registry_artifacts.json.repository | json_query('[*].repository_name') }}"
-
-    - name: Configure K3s node for private registry
-      ansible.builtin.template:
-        dest: /etc/rancher/k3s/registries.yaml
-        src: registries.j2
-      vars:
-        _template:
-          data: "{{ source_registries }}"
-          hv:
-            fqdn: "{{ vapp['metacluster.fqdn'] }}"
+      register: api_readycheck
+      until:
+        - api_readycheck.json.status is defined
+        - api_readycheck.json.status == 'healthy'
+      retries: "{{ playbook.retries }}"
+      delay: "{{ playbook.delays.long }}"
 
   module_defaults:
     ansible.builtin.uri:

ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/storage.yml

@@ -1,14 +1,53 @@
-- name: Increase replicas for each volume
-  kubernetes.core.k8s:
-    api_version: longhorn.io/v1beta2
-    kind: volume
-    name: "{{ item.metadata.name }}"
-    namespace: longhorn-system
-    state: patched
-    definition: |
-      spec:
-        numberOfReplicas: {{ lookup('kubernetes.core.k8s', kind='node', kubeconfig=(kubeconfig.path)) | length | int }}
-    kubeconfig: "{{ kubeconfig.path }}"
-  loop: "{{ lookup('kubernetes.core.k8s', api_version='longhorn.io/v1beta2', kind='volume', namespace='longhorn-system', kubeconfig=(kubeconfig.path)) }}"
-  loop_control:
-    label: "{{ item.metadata.name }}"
+- block:
+
+    - name: Increase replicas for each volume
+      kubernetes.core.k8s:
+        api_version: longhorn.io/v1beta2
+        kind: volume
+        name: "{{ item.metadata.name }}"
+        namespace: longhorn-system
+        state: patched
+        definition: |
+          spec:
+            numberOfReplicas: {{ lookup('kubernetes.core.k8s', kind='node', kubeconfig=(kubeconfig.path)) | length | int }}
+      loop: "{{ lookup('kubernetes.core.k8s', api_version='longhorn.io/v1beta2', kind='volume', namespace='longhorn-system', kubeconfig=(kubeconfig.path)) }}"
+      loop_control:
+        label: "{{ item.metadata.name }}"
+
+    - name: Wait for replica rebuilds to complete
+      ansible.builtin.uri:
+        url: https://storage.{{ vapp['metacluster.fqdn'] }}/v1/volumes
+        method: GET
+      register: volume_details
+      until:
+        - (volume_details.json.data | json_query('[*].robustness') | unique | length) == 1
+        - (volume_details.json.data | json_query('[*].robustness') | first) == "healthy"
+      retries: "{{ playbook.retries }}"
+      delay: "{{ playbook.delays.medium }}"
+
+    - name: Install longhorn chart
+      kubernetes.core.helm:
+        name: longhorn
+        chart_ref: /opt/metacluster/helm-charts/longhorn
+        release_namespace: longhorn-system
+        create_namespace: yes
+        wait: no
+        values: "{{ components.longhorn.chart_values }}"
+
+    - name: Ensure longhorn API availability
+      ansible.builtin.uri:
+        url: https://storage.{{ vapp['metacluster.fqdn'] }}/v1
+        method: GET
+      register: api_readycheck
+      until:
+        - api_readycheck is not failed
+      retries: "{{ playbook.retries }}"
+      delay: "{{ playbook.delays.long }}"
+
+  module_defaults:
+    ansible.builtin.uri:
+      validate_certs: no
+      status_code: [200, 201]
+      body_format: json
+    group/k8s:
+      kubeconfig: "{{ kubeconfig.path }}"