From 4bcb1198f3b251e28a3163bfc7252af0bd0a64bd Mon Sep 17 00:00:00 2001
From: Danny Bessems <danny@bessems.eu>
Date: Wed, 25 Jan 2023 12:36:21 +0100
Subject: [PATCH] Add replica rebuild wait;Upgrade longhorn&harbor

---
 .../metacluster/tasks/containerimages.yml     | 50 ++++++++++++++
 .../upgrade/roles/metacluster/tasks/main.yml  |  6 +-
 .../roles/metacluster/tasks/registry.yml      | 56 +++++-----------
 .../roles/metacluster/tasks/storage.yml       | 67 +++++++++++++++----
 4 files changed, 124 insertions(+), 55 deletions(-)
 create mode 100644 ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/containerimages.yml

diff --git a/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/containerimages.yml b/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/containerimages.yml
new file mode 100644
index 0000000..c0ba9c4
--- /dev/null
+++ b/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/containerimages.yml
@@ -0,0 +1,50 @@
+- block:
+
+    - name: Push images to registry
+      ansible.builtin.shell:
+        cmd: >-
+          skopeo copy \
+            --insecure-policy \
+            --dest-tls-verify=false \
+            --dest-creds admin:{{ vapp['metacluster.password'] }} \
+            docker-archive:./{{ item | basename }} \
+            docker://registry.{{ vapp['metacluster.fqdn'] }}/library/$( \
+              skopeo list-tags \
+                --insecure-policy \
+                docker-archive:./{{ item | basename }} | \
+              jq -r '.Tags[0]')
+        chdir: /opt/metacluster/container-images/
+      register: push_result
+      loop: "{{ query('ansible.builtin.fileglob', '/opt/metacluster/container-images/*.tar') | sort }}"
+      loop_control:
+        label: "{{ item | basename }}"
+      retries: "{{ playbook.retries }}"
+      delay: "{{ playbook.delays.short }}"
+      until: push_result is not failed
+
+    - name: Get all stored container images (=artifacts)
+      ansible.builtin.uri:
+        url: https://registry.{{ vapp['metacluster.fqdn'] }}/api/v2.0/search?q=library
+        method: GET
+      register: registry_artifacts
+
+    - name: Get source registries of all artifacts
+      ansible.builtin.set_fact:
+        source_registries: "{{ (source_registries | default([]) + [(item | split('/'))[1]]) | unique | sort }}"
+      loop: "{{ registry_artifacts.json.repository | json_query('[*].repository_name') }}"
+
+    - name: Configure K3s node for private registry
+      ansible.builtin.template:
+        dest: /etc/rancher/k3s/registries.yaml
+        src: registries.j2
+      vars:
+        _template:
+          data: "{{ source_registries }}"
+          hv:
+            fqdn: "{{ vapp['metacluster.fqdn'] }}"
+
+  module_defaults:
+    ansible.builtin.uri:
+      validate_certs: no
+      status_code: [200, 201, 401]
+      body_format: json
diff --git a/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/main.yml b/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/main.yml
index 8f73c80..3efeca2 100644
--- a/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/main.yml
+++ b/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/main.yml
@@ -1,9 +1,11 @@
 - import_tasks: init.yml
-- import_tasks: registry.yml
+- import_tasks: containerimages.yml
 - import_tasks: k3s.yml
 - import_tasks: assets.yml
-# - import_tasks: ingress.yml
 - import_tasks: storage.yml
+
+# - import_tasks: charts.yml
+- import_tasks: registry.yml
 # - import_tasks: certauthority.yml
 # - import_tasks: git.yml
 # - import_tasks: gitops.yml
diff --git a/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/registry.yml b/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/registry.yml
index c0ba9c4..43a6090 100644
--- a/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/registry.yml
+++ b/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/registry.yml
@@ -1,47 +1,25 @@
 - block:
 
-    - name: Push images to registry
-      ansible.builtin.shell:
-        cmd: >-
-          skopeo copy \
-            --insecure-policy \
-            --dest-tls-verify=false \
-            --dest-creds admin:{{ vapp['metacluster.password'] }} \
-            docker-archive:./{{ item | basename }} \
-            docker://registry.{{ vapp['metacluster.fqdn'] }}/library/$( \
-              skopeo list-tags \
-                --insecure-policy \
-                docker-archive:./{{ item | basename }} | \
-              jq -r '.Tags[0]')
-        chdir: /opt/metacluster/container-images/
-      register: push_result
-      loop: "{{ query('ansible.builtin.fileglob', '/opt/metacluster/container-images/*.tar') | sort }}"
-      loop_control:
-        label: "{{ item | basename }}"
-      retries: "{{ playbook.retries }}"
-      delay: "{{ playbook.delays.short }}"
-      until: push_result is not failed
+    - name: Upgrade harbor chart
+      kubernetes.core.helm:
+        name: harbor
+        chart_ref: /opt/metacluster/helm-charts/harbor
+        release_namespace: harbor
+        create_namespace: yes
+        wait: no
+        kubeconfig: "{{ kubeconfig.path }}"
+        values: "{{ components.harbor.chart_values }}"
 
-    - name: Get all stored container images (=artifacts)
+    - name: Ensure harbor API availability
       ansible.builtin.uri:
-        url: https://registry.{{ vapp['metacluster.fqdn'] }}/api/v2.0/search?q=library
+        url: https://registry.{{ vapp['metacluster.fqdn'] }}/api/v2.0/health
         method: GET
-      register: registry_artifacts
-
-    - name: Get source registries of all artifacts
-      ansible.builtin.set_fact:
-        source_registries: "{{ (source_registries | default([]) + [(item | split('/'))[1]]) | unique | sort }}"
-      loop: "{{ registry_artifacts.json.repository | json_query('[*].repository_name') }}"
-
-    - name: Configure K3s node for private registry
-      ansible.builtin.template:
-        dest: /etc/rancher/k3s/registries.yaml
-        src: registries.j2
-      vars:
-        _template:
-          data: "{{ source_registries }}"
-          hv:
-            fqdn: "{{ vapp['metacluster.fqdn'] }}"
+      register: api_readycheck
+      until:
+        - api_readycheck.json.status is defined
+        - api_readycheck.json.status == 'healthy'
+      retries: "{{ playbook.retries }}"
+      delay: "{{ playbook.delays.long }}"
 
   module_defaults:
     ansible.builtin.uri:
diff --git a/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/storage.yml b/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/storage.yml
index cb76a0e..abea181 100644
--- a/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/storage.yml
+++ b/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/storage.yml
@@ -1,14 +1,53 @@
-- name: Increase replicas for each volume
-  kubernetes.core.k8s:
-    api_version: longhorn.io/v1beta2
-    kind: volume
-    name: "{{ item.metadata.name }}"
-    namespace: longhorn-system
-    state: patched
-    definition: |
-      spec:
-        numberOfReplicas: {{ lookup('kubernetes.core.k8s', kind='node', kubeconfig=(kubeconfig.path)) | length | int }}
-    kubeconfig: "{{ kubeconfig.path }}"
-  loop: "{{ lookup('kubernetes.core.k8s', api_version='longhorn.io/v1beta2', kind='volume', namespace='longhorn-system', kubeconfig=(kubeconfig.path)) }}"
-  loop_control:
-    label: "{{ item.metadata.name }}"
+- block:
+
+    - name: Increase replicas for each volume
+      kubernetes.core.k8s:
+        api_version: longhorn.io/v1beta2
+        kind: volume
+        name: "{{ item.metadata.name }}"
+        namespace: longhorn-system
+        state: patched
+        definition: |
+          spec:
+            numberOfReplicas: {{ lookup('kubernetes.core.k8s', kind='node', kubeconfig=(kubeconfig.path)) | length | int }}
+      loop: "{{ lookup('kubernetes.core.k8s', api_version='longhorn.io/v1beta2', kind='volume', namespace='longhorn-system', kubeconfig=(kubeconfig.path)) }}"
+      loop_control:
+        label: "{{ item.metadata.name }}"
+
+    - name: Wait for replica rebuilds to complete
+      ansible.builtin.uri:
+        url: https://storage.{{ vapp['metacluster.fqdn'] }}/v1/volumes
+        method: GET
+      register: volume_details
+      until:
+        - (volume_details.json.data | json_query('[*].robustness') | unique | length) == 1
+        - (volume_details.json.data | json_query('[*].robustness') | first) == "healthy"
+      retries: "{{ playbook.retries }}"
+      delay: "{{ playbook.delays.medium }}"
+
+    - name: Install longhorn chart
+      kubernetes.core.helm:
+        name: longhorn
+        chart_ref: /opt/metacluster/helm-charts/longhorn
+        release_namespace: longhorn-system
+        create_namespace: yes
+        wait: no
+        values: "{{ components.longhorn.chart_values }}"
+
+    - name: Ensure longhorn API availability
+      ansible.builtin.uri:
+        url: https://storage.{{ vapp['metacluster.fqdn'] }}/v1
+        method: GET
+      register: api_readycheck
+      until:
+        - api_readycheck is not failed
+      retries: "{{ playbook.retries }}"
+      delay: "{{ playbook.delays.long }}"
+
+  module_defaults:
+    ansible.builtin.uri:
+      validate_certs: no
+      status_code: [200, 201]
+      body_format: json
+    group/k8s:
+      kubeconfig: "{{ kubeconfig.path }}"