Add replica rebuild wait;Upgrade longhorn&harbor

ansible/roles/assets/tasks/manifests.yml

@@ -83,4 +83,4 @@
 #     mode: 0600
 #   loop: "{{ lookup('ansible.builtin.dict', components) | map(attribute='value.manifests') | list | select('defined') | flatten }}"
 #   loop_control:
-#     label: "{{ item.type + '/' + item.name }}"
+#     label: "{{ item.type ~ '/' ~ item.name }}"

ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/certauthority.yml

@@ -62,7 +62,7 @@
             - key: root_ca.crt
               value: "{{ stepca_cm_certs.resources[0].data['root_ca.crt'] | b64encode }}"
       loop_control:
-        label: "{{ item.kind + '/' + item.name + ' (' + item.namespace + ')' }}"
+        label: "{{ item.kind ~ '/' ~ item.name ~ ' (' ~ item.namespace ~ ')' }}"
 
     - name: Configure step-ca passthrough ingress
       ansible.builtin.template:

ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/git.yml

@@ -120,7 +120,7 @@
                 # default_branch: main
                 description: GitOps manifests
           loop_control:
-            label: "{{ item.organization + '/' + item.body.name }}"
+            label: "{{ item.organization ~ '/' ~ item.body.name }}"
 
         - name: Rebase/Push source gitops repository
           ansible.builtin.shell:

ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/init.yml

@@ -1,7 +1,7 @@
 - name: Configure fallback name resolution
   ansible.builtin.lineinfile:
     path: /etc/hosts
-    line: "{{ vapp['guestinfo.ipaddress'] }}  {{ item + '.' + vapp['metacluster.fqdn'] }}"
+    line: "{{ vapp['guestinfo.ipaddress'] }}  {{ item ~ '.' ~ vapp['metacluster.fqdn'] }}"
     state: present
   loop:
     # TODO: Make this list dynamic

ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/clusterapi.yml

@@ -1,4 +1,7 @@
 - block:
+  # Below tasks circumvent usernames with `<domain>\<username>` format, which causes CAPV to
+  # incorrectly interpret the backslash (despite automatic escaping) as an escape sequence.
+  # `vcenter_session.user` will instead contain the username in `<username>@<domain>` format.
 
     - name: Generate vCenter API token
       ansible.builtin.uri:
@@ -13,7 +16,7 @@
         url: https://{{ vapp['hv.fqdn'] }}/api/session
         method: GET
         headers:
-          vmware-api-session-id: "{{ vcenter_api_token.json }}"
+          vmware-api-session-id: "{{ vcenterapi_token.json }}"
       register: vcenter_session
 
   module_defaults:
@@ -51,7 +54,7 @@
 
 - name: Update image references to use local registry
   ansible.builtin.replace:
-    dest: "{{ item.root + '/' + item.path }}"
+    dest: "{{ item.root ~ '/' ~ item.path }}"
     regexp: '([ ]+image:[ "]+)(?!({{ _template.pattern }}|"{{ _template.pattern }}))'
     replace: '\1{{ _template.pattern }}'
   vars:
@@ -82,7 +85,7 @@
 - name: Store custom cluster-template
   ansible.builtin.copy:
     dest: /opt/metacluster/cluster-api/custom-cluster-template.yaml
-    content: "{{ lookup('kubernetes.core.kustomize', dir='/opt/metacluster/cluster-api/infrastructure-vsphere/' + components.clusterapi.management.version.infrastructure_vsphere ) }}"
+    content: "{{ lookup('kubernetes.core.kustomize', dir='/opt/metacluster/cluster-api/infrastructure-vsphere/' ~ components.clusterapi.management.version.infrastructure_vsphere ) }}"
 
 - name: Initialize Cluster API management cluster
   ansible.builtin.shell:

ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/hypervisor.yml

@@ -68,7 +68,7 @@
         --portgroup "{{ vcenter_info.network }}" \
         --startaddress {{ vapp['ippool.startip'] }} \
         --endaddress {{ vapp['ippool.endip'] }} \
-        --netmask {{ (vapp['guestinfo.ipaddress'] + '/' + vapp['guestinfo.prefixlength']) | ansible.utils.ipaddr('netmask') }} \
+        --netmask {{ (vapp['guestinfo.ipaddress'] ~ '/' ~ vapp['guestinfo.prefixlength']) | ansible.utils.ipaddr('netmask') }} \
         {{ vapp['guestinfo.dnsserver'] | split(',') | map('trim') | map('regex_replace', '^', '--dnsserver ') | join(' ') }} \
         --dnsdomain {{ vapp['metacluster.fqdn'] }} \
         --gateway {{ vapp['guestinfo.gateway'] }} \

ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/registry.yml

@@ -5,7 +5,7 @@
       url: https://registry.{{ vapp['metacluster.fqdn'] }}/api/v2.0/projects
       method: POST
       headers:
-        Authorization: "Basic {{ ('admin:' + vapp['metacluster.password']) | b64encode }}"
+        Authorization: "Basic {{ ('admin:' ~ vapp['metacluster.password']) | b64encode }}"
       body:
         project_name: kubeadm
         public: true
@@ -28,7 +28,7 @@
       url: https://registry.{{ vapp['metacluster.fqdn'] }}/api/v2.0/projects/kubeadm/repositories/{{ ( item | regex_findall('([^:/]+)') )[-2] }}/artifacts?from=library/{{ item | replace('/', '%2F') | replace(':', '%3A') }}
       method: POST
       headers:
-        Authorization: "Basic {{ ('admin:' + vapp['metacluster.password']) | b64encode }}"
+        Authorization: "Basic {{ ('admin:' ~ vapp['metacluster.password']) | b64encode }}"
       body:
         from: "{{ item }}"
     loop: "{{ kubeadm_images }}"

ansible/roles/firstboot/files/ansible_payload/common/roles/users/tasks/main.yml

@@ -25,7 +25,7 @@
     line: 'PasswordAuthentication yes'
     state: absent
   loop_control:
-    label: "{{ '[' + item.regex + '] ' + item.state }}"
+    label: "{{ '[' ~ item.regex ~ '] ' ~ item.state }}"
 
 - name: Create dedicated SSH keypair
   community.crypto.openssh_keypair:

ansible/roles/firstboot/files/ansible_payload/common/templates/tty.j2

@@ -17,7 +17,7 @@ COMPONENTS=('storage' 'registry' 'git' 'gitops')
 FQDN='{{ _template.metacluster.fqdn }}'
 IPADDRESS='{{ _template.metacluster.vip }}'
 
-I=60
+I=0
 
 while /bin/true; do
   if [[ $I -gt 59 ]]; then

ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/containerimages.yml

@@ -0,0 +1,50 @@
+- block:
+
+    - name: Push images to registry
+      ansible.builtin.shell:
+        cmd: >-
+          skopeo copy \
+            --insecure-policy \
+            --dest-tls-verify=false \
+            --dest-creds admin:{{ vapp['metacluster.password'] }} \
+            docker-archive:./{{ item | basename }} \
+            docker://registry.{{ vapp['metacluster.fqdn'] }}/library/$( \
+              skopeo list-tags \
+                --insecure-policy \
+                docker-archive:./{{ item | basename }} | \
+              jq -r '.Tags[0]')
+        chdir: /opt/metacluster/container-images/
+      register: push_result
+      loop: "{{ query('ansible.builtin.fileglob', '/opt/metacluster/container-images/*.tar') | sort }}"
+      loop_control:
+        label: "{{ item | basename }}"
+      retries: "{{ playbook.retries }}"
+      delay: "{{ playbook.delays.short }}"
+      until: push_result is not failed
+
+    - name: Get all stored container images (=artifacts)
+      ansible.builtin.uri:
+        url: https://registry.{{ vapp['metacluster.fqdn'] }}/api/v2.0/search?q=library
+        method: GET
+      register: registry_artifacts
+
+    - name: Get source registries of all artifacts
+      ansible.builtin.set_fact:
+        source_registries: "{{ (source_registries | default([]) + [(item | split('/'))[1]]) | unique | sort }}"
+      loop: "{{ registry_artifacts.json.repository | json_query('[*].repository_name') }}"
+
+    - name: Configure K3s node for private registry
+      ansible.builtin.template:
+        dest: /etc/rancher/k3s/registries.yaml
+        src: registries.j2
+      vars:
+        _template:
+          data: "{{ source_registries }}"
+          hv:
+            fqdn: "{{ vapp['metacluster.fqdn'] }}"
+
+  module_defaults:
+    ansible.builtin.uri:
+      validate_certs: no
+      status_code: [200, 201, 401]
+      body_format: json

ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/init.yml

@@ -1,7 +1,7 @@
 - name: Configure fallback name resolution
   ansible.builtin.lineinfile:
     path: /etc/hosts
-    line: "{{ vapp['metacluster.vip'] }}  {{ item + '.' + vapp['metacluster.fqdn'] }}"
+    line: "{{ vapp['metacluster.vip'] }}  {{ item ~ '.' ~ vapp['metacluster.fqdn'] }}"
     state: present
   loop:
     # TODO: Make this list dynamic

ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/main.yml

@@ -1,9 +1,11 @@
 - import_tasks: init.yml
-- import_tasks: registry.yml
+- import_tasks: containerimages.yml
 - import_tasks: k3s.yml
 - import_tasks: assets.yml
-# - import_tasks: ingress.yml
 - import_tasks: storage.yml
+
+# - import_tasks: charts.yml
+- import_tasks: registry.yml
 # - import_tasks: certauthority.yml
 # - import_tasks: git.yml
 # - import_tasks: gitops.yml

ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/registry.yml

@@ -1,47 +1,25 @@
 - block:
 
-    - name: Push images to registry
-      ansible.builtin.shell:
-        cmd: >-
-          skopeo copy \
-            --insecure-policy \
-            --dest-tls-verify=false \
-            --dest-creds admin:{{ vapp['metacluster.password'] }} \
-            docker-archive:./{{ item | basename }} \
-            docker://registry.{{ vapp['metacluster.fqdn'] }}/library/$( \
-              skopeo list-tags \
-                --insecure-policy \
-                docker-archive:./{{ item | basename }} | \
-              jq -r '.Tags[0]')
-        chdir: /opt/metacluster/container-images/
-      register: push_result
-      loop: "{{ query('ansible.builtin.fileglob', '/opt/metacluster/container-images/*.tar') | sort }}"
-      loop_control:
-        label: "{{ item | basename }}"
-      retries: "{{ playbook.retries }}"
-      delay: "{{ playbook.delays.short }}"
-      until: push_result is not failed
+    - name: Upgrade harbor chart
+      kubernetes.core.helm:
+        name: harbor
+        chart_ref: /opt/metacluster/helm-charts/harbor
+        release_namespace: harbor
+        create_namespace: yes
+        wait: no
+        kubeconfig: "{{ kubeconfig.path }}"
+        values: "{{ components.harbor.chart_values }}"
 
-    - name: Get all stored container images (=artifacts)
+    - name: Ensure harbor API availability
       ansible.builtin.uri:
-        url: https://registry.{{ vapp['metacluster.fqdn'] }}/api/v2.0/search?q=library
+        url: https://registry.{{ vapp['metacluster.fqdn'] }}/api/v2.0/health
         method: GET
-      register: registry_artifacts
-
-    - name: Get source registries of all artifacts
-      ansible.builtin.set_fact:
-        source_registries: "{{ (source_registries | default([]) + [(item | split('/'))[1]]) | unique | sort }}"
-      loop: "{{ registry_artifacts.json.repository | json_query('[*].repository_name') }}"
-
-    - name: Configure K3s node for private registry
-      ansible.builtin.template:
-        dest: /etc/rancher/k3s/registries.yaml
-        src: registries.j2
-      vars:
-        _template:
-          data: "{{ source_registries }}"
-          hv:
-            fqdn: "{{ vapp['metacluster.fqdn'] }}"
+      register: api_readycheck
+      until:
+        - api_readycheck.json.status is defined
+        - api_readycheck.json.status == 'healthy'
+      retries: "{{ playbook.retries }}"
+      delay: "{{ playbook.delays.long }}"
 
   module_defaults:
     ansible.builtin.uri:

ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/storage.yml

@@ -1,3 +1,5 @@
+- block:
+
     - name: Increase replicas for each volume
       kubernetes.core.k8s:
         api_version: longhorn.io/v1beta2
@@ -8,7 +10,44 @@
         definition: |
           spec:
             numberOfReplicas: {{ lookup('kubernetes.core.k8s', kind='node', kubeconfig=(kubeconfig.path)) | length | int }}
-    kubeconfig: "{{ kubeconfig.path }}"
       loop: "{{ lookup('kubernetes.core.k8s', api_version='longhorn.io/v1beta2', kind='volume', namespace='longhorn-system', kubeconfig=(kubeconfig.path)) }}"
       loop_control:
         label: "{{ item.metadata.name }}"
+
+    - name: Wait for replica rebuilds to complete
+      ansible.builtin.uri:
+        url: https://storage.{{ vapp['metacluster.fqdn'] }}/v1/volumes
+        method: GET
+      register: volume_details
+      until:
+        - (volume_details.json.data | json_query('[*].robustness') | unique | length) == 1
+        - (volume_details.json.data | json_query('[*].robustness') | first) == "healthy"
+      retries: "{{ playbook.retries }}"
+      delay: "{{ playbook.delays.medium }}"
+
+    - name: Install longhorn chart
+      kubernetes.core.helm:
+        name: longhorn
+        chart_ref: /opt/metacluster/helm-charts/longhorn
+        release_namespace: longhorn-system
+        create_namespace: yes
+        wait: no
+        values: "{{ components.longhorn.chart_values }}"
+
+    - name: Ensure longhorn API availability
+      ansible.builtin.uri:
+        url: https://storage.{{ vapp['metacluster.fqdn'] }}/v1
+        method: GET
+      register: api_readycheck
+      until:
+        - api_readycheck is not failed
+      retries: "{{ playbook.retries }}"
+      delay: "{{ playbook.delays.long }}"
+
+  module_defaults:
+    ansible.builtin.uri:
+      validate_certs: no
+      status_code: [200, 201]
+      body_format: json
+    group/k8s:
+      kubeconfig: "{{ kubeconfig.path }}"

ansible/roles/firstboot/templates/firstboot.j2

@@ -1,4 +1,10 @@
 #!/bin/bash
 
+# Workaround for ansible output regression
+export PYTHONUNBUFFERED=1
+
 # Apply firstboot configuration w/ ansible
-/usr/local/bin/ansible-playbook -e "PYTHONUNBUFFERED=1" /opt/firstboot/ansible/playbook.yml | tee -a /var/log/firstboot.log > /dev/tty1 2>&1
+/usr/local/bin/ansible-playbook /opt/firstboot/ansible/playbook.yml | tee -a /var/log/firstboot.log > /dev/tty1 2>&1
+
+# Cleanup console
+clear > /dev/tty1

ansible/vars/metacluster.yml

@@ -42,6 +42,9 @@ platform:
               websecure:
                 tls:
                   certResolver: stepca
+            updateStrategy:
+              type: Recreate
+              rollingUpdate: null
 
   helm_repositories:
     - name: argo

scripts/Update-OvfConfiguration.bootstrap.yml

@@ -106,13 +106,13 @@ PropertyCategories:
     Configurations: '*'
     UserConfigurable: true
 
-  - Key: guestinfo.ntpserver
-    Type: string(1..)
-    Label: Time server*
-    Description: A comma-separated list of timeservers
-    DefaultValue: 0.pool.ntp.org,1.pool.ntp.org,2.pool.ntp.org
-    Configurations: '*'
-    UserConfigurable: true
+  # - Key: guestinfo.ntpserver
+  #   Type: string(1..)
+  #   Label: Time server*
+  #   Description: A comma-separated list of timeservers
+  #   DefaultValue: 0.pool.ntp.org,1.pool.ntp.org,2.pool.ntp.org
+  #   Configurations: '*'
+  #   UserConfigurable: true
 
 - Name: 3) Workload-cluster
   ProductProperties:

scripts/Update-OvfConfiguration.upgrade.yml

@@ -87,13 +87,13 @@ PropertyCategories:
     Configurations: '*'
     UserConfigurable: true
 
-  - Key: guestinfo.ntpserver
-    Type: string(1..)
-    Label: Time server*
-    Description: A comma-separated list of timeservers
-    DefaultValue: 0.pool.ntp.org,1.pool.ntp.org,2.pool.ntp.org
-    Configurations: '*'
-    UserConfigurable: true
+  # - Key: guestinfo.ntpserver
+  #   Type: string(1..)
+  #   Label: Time server*
+  #   Description: A comma-separated list of timeservers
+  #   DefaultValue: 0.pool.ntp.org,1.pool.ntp.org,2.pool.ntp.org
+  #   Configurations: '*'
+  #   UserConfigurable: true
 
 - Name: 3) Common
   ProductProperties: