fix: Incorrect indentation causing malformed PEM file

.drone.yml

@@ -13,12 +13,42 @@ volumes:
 steps:
 - name: Debugging information
   image: bv11-cr01.bessems.eu/library/packer-extended
+  pull: always
   commands:
   - ansible --version
   - ovftool --version
   - packer --version
   - yamllint --version
+
+- name: Linting
+  depends_on:
+  - Debugging information
+  image: bv11-cr01.bessems.eu/library/packer-extended
+  pull: always
+  commands:
+  - |
+    yamllint -d "{extends: relaxed, rules: {line-length: disable}}" \
+      ansible \
+      packer/preseed/UbuntuServer22.04/user-data \
+      scripts
+
+- name: Install Ansible Galaxy collections
+  depends_on:
+  - Linting
+  image: bv11-cr01.bessems.eu/library/packer-extended
+  pull: always
+  commands:
+  - |
+    ansible-galaxy collection install \
+      -r ansible/requirements.yml \
+      -p ./ansible/collections
+  volumes:
+  - name: scratch
+    path: /scratch
+
 - name: Kubernetes Bootstrap Appliance
+  depends_on:
+  - Install Ansible Galaxy collections
   image: bv11-cr01.bessems.eu/library/packer-extended
   pull: always
   commands:
@@ -26,47 +56,118 @@ steps:
     sed -i -e "s/<<img-password>>/$${SSH_PASSWORD}/g" \
       packer/preseed/UbuntuServer22.04/user-data
   - |
-    yamllint -d "{extends: relaxed, rules: {line-length: disable}}" \
-      ansible \
-      packer/preseed/UbuntuServer22.04/user-data \
-      scripts
-  - |
-    ansible-galaxy install \
-      -r ansible/requirements.yml
+    export K8S_VERSION=$(yq '.components.clusterapi.workload.version.k8s' < ./ansible/vars/metacluster.yml)
   - |
     packer init -upgrade \
       ./packer
   - |
     packer validate \
-      -var vm_name=$DRONE_BUILD_NUMBER-${DRONE_COMMIT_SHA:0:10} \
+      -only=vsphere-iso.bootstrap \
+      -var vm_name=${DRONE_BUILD_NUMBER}-${DRONE_COMMIT_SHA:0:10}-$(openssl rand -hex 3) \
+      -var docker_username=$${DOCKER_USERNAME} \
+      -var docker_password=$${DOCKER_PASSWORD} \
       -var repo_username=$${REPO_USERNAME} \
       -var repo_password=$${REPO_PASSWORD} \
-      -var vsphere_password=$${VSPHERE_PASSWORD} \
       -var ssh_password=$${SSH_PASSWORD} \
+      -var vsphere_password=$${VSPHERE_PASSWORD} \
+      -var k8s_version=$K8S_VERSION \
       ./packer
   - |
     packer build \
       -on-error=cleanup -timestamp-ui \
-      -var vm_name=$DRONE_BUILD_NUMBER-${DRONE_COMMIT_SHA:0:10} \
+      -only=vsphere-iso.bootstrap \
+      -var vm_name=${DRONE_BUILD_NUMBER}-${DRONE_COMMIT_SHA:0:10}-$(openssl rand -hex 3) \
+      -var docker_username=$${DOCKER_USERNAME} \
+      -var docker_password=$${DOCKER_PASSWORD} \
       -var repo_username=$${REPO_USERNAME} \
       -var repo_password=$${REPO_PASSWORD} \
-      -var vsphere_password=$${VSPHERE_PASSWORD} \
       -var ssh_password=$${SSH_PASSWORD} \
+      -var vsphere_password=$${VSPHERE_PASSWORD} \
+      -var k8s_version=$K8S_VERSION \
       ./packer
   environment:
-    VSPHERE_PASSWORD:
-      from_secret: vsphere_password
-    SSH_PASSWORD:
-      from_secret: ssh_password
+    DOCKER_USERNAME:
+      from_secret: docker_username
+    DOCKER_PASSWORD:
+      from_secret: docker_password
+    # PACKER_LOG: 1
     REPO_USERNAME:
       from_secret: repo_username
     REPO_PASSWORD:
       from_secret: repo_password
-    # PACKER_LOG: 1
+    SSH_PASSWORD:
+      from_secret: ssh_password
+    VSPHERE_PASSWORD:
+      from_secret: vsphere_password
   volumes:
   - name: output
     path: /output
+  - name: scratch
+    path: /scratch
+
+- name: Kubernetes Upgrade Appliance
+  depends_on:
+  - Install Ansible Galaxy collections
+  image: bv11-cr01.bessems.eu/library/packer-extended
+  pull: alwaysquery(
+  commands:
+  - |
+    sed -i -e "s/<<img-password>>/$${SSH_PASSWORD}/g" \
+      packer/preseed/UbuntuServer22.04/user-data
+  - |
+    export K8S_VERSION=$(yq '.components.clusterapi.workload.version.k8s' < ./ansible/vars/metacluster.yml)
+  - |
+    packer init -upgrade \
+      ./packer
+  - |
+    packer validate \
+      -only=vsphere-iso.upgrade \
+      -var vm_name=${DRONE_BUILD_NUMBER}-${DRONE_COMMIT_SHA:0:10}-$(openssl rand -hex 3) \
+      -var docker_username=$${DOCKER_USERNAME} \
+      -var docker_password=$${DOCKER_PASSWORD} \
+      -var repo_username=$${REPO_USERNAME} \
+      -var repo_password=$${REPO_PASSWORD} \
+      -var ssh_password=$${SSH_PASSWORD} \
+      -var vsphere_password=$${VSPHERE_PASSWORD} \
+      -var k8s_version=$K8S_VERSION \
+      ./packer
+  - |
+    packer build \
+      -on-error=cleanup -timestamp-ui \
+      -only=vsphere-iso.upgrade \
+      -var vm_name=${DRONE_BUILD_NUMBER}-${DRONE_COMMIT_SHA:0:10}-$(openssl rand -hex 3) \
+      -var docker_username=$${DOCKER_USERNAME} \
+      -var docker_password=$${DOCKER_PASSWORD} \
+      -var repo_username=$${REPO_USERNAME} \
+      -var repo_password=$${REPO_PASSWORD} \
+      -var ssh_password=$${SSH_PASSWORD} \
+      -var vsphere_password=$${VSPHERE_PASSWORD} \
+      -var k8s_version=$K8S_VERSION \
+      ./packer
+  environment:
+    DOCKER_USERNAME:
+      from_secret: docker_username
+    DOCKER_PASSWORD:
+      from_secret: docker_password
+    # PACKER_LOG: 1
+    REPO_USERNAME:
+      from_secret: repo_username
+    REPO_PASSWORD:
+      from_secret: repo_password
+    SSH_PASSWORD:
+      from_secret: ssh_password
+    VSPHERE_PASSWORD:
+      from_secret: vsphere_password
+  volumes:
+  - name: output
+    path: /output
+  - name: scratch
+    path: /scratch
+
 - name: Remove temporary resources
+  depends_on:
+  - Kubernetes Bootstrap Appliance
+  - Kubernetes Upgrade Appliance
   image: bv11-cr01.bessems.eu/library/packer-extended
   commands:
   - |

ansible/playbook.yml

@@ -3,6 +3,7 @@
   gather_facts: false
   vars_files:
     - metacluster.yml
+    - workloadcluster.yml
   become: true
   roles:
     - os

ansible/requirements.yml

@@ -1,4 +1,6 @@
 collections:
+  - name: https://github.com/ansible-collections/ansible.posix
+    type: git
   - name: https://github.com/ansible-collections/ansible.utils
     type: git
   - name: https://github.com/ansible-collections/community.general

ansible/roles/assets/tasks/containerimages.yml

@@ -14,13 +14,22 @@
   loop_control:
     label: "{{ item.dest | basename }}"
 
-- name: Parse helm charts for container images
+- name: Parse metacluster helm charts for container images
   ansible.builtin.shell:
     cmd: "{{ item.value.helm.parse_logic }}"
     chdir: /opt/metacluster/helm-charts/{{ item.key }}
-  register: chartimages
+  register: chartimages_metacluster
   when: item.value.helm is defined
-  loop: "{{ lookup('ansible.builtin.dict', components) }}"
+  loop: "{{ query('ansible.builtin.dict', components) }}"
+  loop_control:
+    label: "{{ item.key }}"
+
+- name: Parse workloadcluster helm charts for container images
+  ansible.builtin.shell:
+    cmd: "{{ item.value.parse_logic }}"
+    chdir: /opt/workloadcluster/helm-charts/{{ item.value.namespace }}/{{ item.key }}
+  register: chartimages_workloadcluster
+  loop: "{{ query('ansible.builtin.dict', downstream.helm_charts) }}"
   loop_control:
     label: "{{ item.key }}"
 
@@ -29,7 +38,7 @@
     containerimages_{{ item.source }}: "{{ item.results }}"
   loop:
     - source: charts
-      results: "{{ chartimages | json_query('results[*].stdout_lines') | select() | flatten | list }}"
+      results: "{{ (chartimages_metacluster | json_query('results[*].stdout_lines')) + (chartimages_workloadcluster | json_query('results[*].stdout_lines')) | select() | flatten | list }}"
     - source: kubeadm
       results: "{{ kubeadmimages.stdout_lines }}"
     - source: manifests
@@ -37,6 +46,15 @@
   loop_control:
     label: "{{ item.source }}"
 
+- name: Log in to container registry
+  ansible.builtin.shell:
+    cmd: >-
+      skopeo login \
+        docker.io \
+        --username={{ docker_username }} \
+        --password={{ docker_password }}
+  no_log: true
+
 - name: Pull and store containerimages
   ansible.builtin.shell:
     cmd: >-

ansible/roles/assets/tasks/git.yml

@@ -1,5 +0,0 @@
-- name: Clone git repository
-  ansible.builtin.git:
-    repo: "{{ platform.gitops.repository.uri }}"
-    version: "{{ platform.gitops.repository.revision }}"
-    dest: /opt/metacluster/git-repositories/gitops

ansible/roles/assets/tasks/helm.yml

@@ -3,17 +3,29 @@
     name: "{{ item.name }}"
     repo_url: "{{ item.url }}"
     state: present
-  loop: "{{ platform.helm_repositories }}"
+  loop: "{{ platform.helm_repositories + downstream.helm_repositories }}"
 
-- name: Fetch helm charts
+- name: Fetch helm charts for metacluster
   ansible.builtin.command:
     cmd: helm fetch {{ item.value.helm.chart }} --untar --version {{ item.value.helm.version }}
     chdir: /opt/metacluster/helm-charts
   when: item.value.helm is defined
-  register: helmcharts
-  loop: "{{ lookup('ansible.builtin.dict', components) }}"
+  register: helmcharts_metacluster
+  loop: "{{ query('ansible.builtin.dict', components) }}"
   loop_control:
     label: "{{ item.key }}"
   retries: 5
   delay: 5
-  until: helmcharts is not failed
+  until: helmcharts_metacluster is not failed
+
+- name: Fetch helm charts for workloadcluster
+  ansible.builtin.command:
+    cmd: helm fetch {{ item.value.chart }} --untardir ./{{ item.value.namespace }} --untar --version {{ item.value.version }}
+    chdir: /opt/workloadcluster/helm-charts
+  register: helmcharts_workloadcluster
+  loop: "{{ query('ansible.builtin.dict', downstream.helm_charts) }}"
+  loop_control:
+    label: "{{ item.key }}"
+  retries: 5
+  delay: 5
+  until: helmcharts_workloadcluster is not failed

ansible/roles/assets/tasks/k3s.yml

@@ -21,7 +21,7 @@
 
 - name: Download K3s install script
   ansible.builtin.get_url:
-    url: https://get.k3s.io
+    url: https://raw.githubusercontent.com/k3s-io/k3s/{{ platform.k3s.version | urlencode }}/install.sh
     dest: /opt/metacluster/k3s/install.sh
     owner: root
     group: root

ansible/roles/assets/tasks/main.yml

@@ -12,10 +12,13 @@
     - /opt/metacluster/cluster-api/infrastructure-vsphere/{{ components.clusterapi.management.version.infrastructure_vsphere }}
     - /opt/metacluster/cluster-api/ipam-in-cluster/{{ components.clusterapi.management.version.ipam_incluster }}
     - /opt/metacluster/container-images
-    - /opt/metacluster/git-repositories/gitops
+    - /opt/metacluster/git-repositories
     - /opt/metacluster/helm-charts
     - /opt/metacluster/k3s
     - /opt/metacluster/kube-vip
+    - /opt/workloadcluster/git-repositories/gitops/charts
+    - /opt/workloadcluster/git-repositories/gitops/values
+    - /opt/workloadcluster/helm-charts
     - /opt/workloadcluster/node-templates
     - /var/lib/rancher/k3s/agent/images
     - /var/lib/rancher/k3s/server/manifests
@@ -23,7 +26,7 @@
 - import_tasks: dependencies.yml
 - import_tasks: k3s.yml
 - import_tasks: helm.yml
-- import_tasks: git.yml
+# - import_tasks: git.yml
 - import_tasks: manifests.yml
 - import_tasks: kubeadm.yml
 - import_tasks: containerimages.yml

ansible/roles/assets/tasks/manifests.yml

@@ -2,9 +2,9 @@
 
     - name: Aggregate chart_values into dict
       ansible.builtin.set_fact:
-        chart_values: "{{ chart_values | default({}) | combine({ (item.key | regex_replace('[^A-Za-z0-9]', '')): { 'chart_values': (item.value.helm.chart_values | from_yaml) } }) }}"
+        metacluster_chartvalues: "{{ metacluster_chartvalues | default({}) | combine({ item.key: { 'chart_values': (item.value.helm.chart_values | from_yaml) } }) }}"
       when: item.value.helm.chart_values is defined
-      loop: "{{ lookup('ansible.builtin.dict', components) }}"
+      loop: "{{ query('ansible.builtin.dict', components) }}"
       loop_control:
         label: "{{ item.key }}"
 
@@ -14,12 +14,29 @@
         content: >-
           {{
             { 'components': (
-              chart_values |
+              metacluster_chartvalues |
               combine({ 'clusterapi': components.clusterapi }) |
               combine({ 'kubevip'   : components.kubevip }) )
             } | to_nice_yaml(indent=2, width=4096)
           }}
 
+    - name: Aggregate chart_values into dict
+      ansible.builtin.set_fact:
+        workloadcluster_chartvalues: "{{ workloadcluster_chartvalues | default({}) | combine({ item.key: { 'chart_values': (item.value.chart_values | default('') | from_yaml) } }) }}"
+      # when: item.value.chart_values is defined
+      loop: "{{ query('ansible.builtin.dict', downstream.helm_charts) }}"
+      loop_control:
+        label: "{{ item.key }}"
+
+    - name: Write dict to vars_file
+      ansible.builtin.copy:
+        dest: /opt/firstboot/ansible/vars/workloadcluster.yml
+        content: >-
+          {{
+            { 'downstream_components': ( workloadcluster_chartvalues )
+            } | to_nice_yaml(indent=2, width=4096)
+          }}
+
 - name: Download ClusterAPI manifests
   ansible.builtin.get_url:
     url: "{{ item.url }}"
@@ -65,6 +82,12 @@
   delay: 5
   until: clusterapi_manifests is not failed
 
+- name: Update cluster-template with image tags
+  ansible.builtin.replace:
+    dest: /opt/metacluster/cluster-api/infrastructure-vsphere/{{ components.clusterapi.management.version.infrastructure_vsphere }}/cluster-template.yaml
+    regexp: ':\${CPI_IMAGE_K8S_VERSION}'
+    replace: ":{{ components.clusterapi.management.version.cpi_vsphere }}"
+
 - name: Download kube-vip RBAC manifest
   ansible.builtin.get_url:
     url: https://kube-vip.io/manifests/rbac.yaml
@@ -81,6 +104,6 @@
 #     owner: root
 #     group: root
 #     mode: 0600
-#   loop: "{{ lookup('ansible.builtin.dict', components) | map(attribute='value.manifests') | list | select('defined') | flatten }}"
+#   loop: "{{ query('ansible.builtin.dict', components) | map(attribute='value.manifests') | list | select('defined') | flatten }}"
 #   loop_control:
-#     label: "{{ item.type + '/' + item.name }}"
+#     label: "{{ item.type ~ '/' ~ item.name }}"

ansible/roles/firstboot/files/ansible_payload/bootstrap/playbook.yml

@@ -5,6 +5,7 @@
   vars_files:
     - defaults.yml
     - metacluster.yml
+    - workloadcluster.yml
   # become: true
   roles:
     - vapp

ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/filter_plugins/netaddr.py

@@ -1,14 +0,0 @@
-import netaddr
-
-def netaddr_iter_iprange(ip_start, ip_end):
-    return [str(ip) for ip in netaddr.iter_iprange(ip_start, ip_end)]
-
-class FilterModule(object):
-        ''' Ansible filter. Interface to netaddr methods.
-            https://pypi.org/project/netaddr/
-        '''
-
-        def filters(self):
-            return {
-                'netaddr_iter_iprange': netaddr_iter_iprange
-            }

ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/certauthority.yml

@@ -1,15 +1,33 @@
 - block:
 
+    - name: Initialize tempfile
+      ansible.builtin.tempfile:
+        state: file
+      register: values_file
+
+    - name: Write chart values w/ password to tempfile
+      ansible.builtin.copy:
+        dest: "{{ values_file.path }}"
+        content: "{{ stepca_values.stdout | regex_replace('(ca_password|provisioner_password): ', '\\1: ' ~ (vapp['metacluster.password'] | b64encode)) }}"
+      no_log: true
+
     - name: Install step-ca chart
       kubernetes.core.helm:
         name: step-certificates
         chart_ref: /opt/metacluster/helm-charts/step-certificates
         release_namespace: step-ca
-        create_namespace: yes
-        # Unable to use REST api based readycheck due to missing ingress
-        wait: yes
+        create_namespace: true
+        # Unable to use REST api based readycheck due to lack of ingress
+        wait: true
         kubeconfig: "{{ kubeconfig.path }}"
-        values: "{{ components.stepcertificates.chart_values }}"
+        values_files:
+          - "{{ values_file.path }}"
+
+    - name: Cleanup tempfile
+      ansible.builtin.file:
+        path: "{{ values_file.path }}"
+        state: absent
+      when: values_file.path is defined
 
     - name: Retrieve configmap w/ root certificate
       kubernetes.core.k8s_info:
@@ -62,7 +80,7 @@
             - key: root_ca.crt
               value: "{{ stepca_cm_certs.resources[0].data['root_ca.crt'] | b64encode }}"
       loop_control:
-        label: "{{ item.kind + '/' + item.name + ' (' + item.namespace + ')' }}"
+        label: "{{ item.kind ~ '/' ~ item.name ~ ' (' ~ item.namespace ~ ')' }}"
 
     - name: Configure step-ca passthrough ingress
       ansible.builtin.template:
@@ -99,30 +117,23 @@
               env:
                 - name: LEGO_CA_CERTIFICATES
                   value: /step-ca/root_ca.crt
-        marker: '    # {mark} ANSIBLE MANAGED BLOCK'
+        marker: '    # {mark} ANSIBLE MANAGED BLOCK [rootca]'
       notify:
         - Apply manifests
 
     - name: Trigger handlers
       ansible.builtin.meta: flush_handlers
 
-    - name: Retrieve step-ca configuration
-      kubernetes.core.k8s_info:
-        kind: ConfigMap
-        name: step-certificates-config
-        namespace: step-ca
-        kubeconfig: "{{ kubeconfig.path }}"
-      register: stepca_cm_config
-
-    - name: Install root CA in system truststore
-      ansible.builtin.shell:
-        cmd: >-
-          step ca bootstrap \
-            --ca-url=https://ca.{{ vapp['metacluster.fqdn'] }} \
-            --fingerprint={{ stepca_cm_config.resources[0].data['defaults.json'] | from_json | json_query('fingerprint') }} \
-            --install \
-            --force
-          update-ca-certificates
+    - name: Ensure step-ca API availability
+      ansible.builtin.uri:
+        url: https://ca.{{ vapp['metacluster.fqdn'] }}/health
+        method: GET
+      register: api_readycheck
+      until:
+        - api_readycheck.json.status is defined
+        - api_readycheck.json.status == 'ok'
+      retries: "{{ playbook.retries }}"
+      delay: "{{ (storage_benchmark | int) * (playbook.delay.long | int) }}"
 
   module_defaults:
     ansible.builtin.uri:

ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/git.yml

@@ -5,10 +5,10 @@
         name: gitea
         chart_ref: /opt/metacluster/helm-charts/gitea
         release_namespace: gitea
-        create_namespace: yes
-        wait: no
+        create_namespace: true
+        wait: false
         kubeconfig: "{{ kubeconfig.path }}"
-        values: "{{ components.gitea.chart_values }}"
+        values: "{{ components['gitea'].chart_values }}"
 
     - name: Ensure gitea API availability
       ansible.builtin.uri:
@@ -19,7 +19,7 @@
         - api_readycheck.json.status is defined
         - api_readycheck.json.status == 'pass'
       retries: "{{ playbook.retries }}"
-      delay: "{{ playbook.delays.long }}"
+      delay: "{{ (storage_benchmark | int) * (playbook.delay.long | int) }}"
 
     - name: Configure additional SSH ingress
       ansible.builtin.template:
@@ -107,6 +107,12 @@
               Authorization: token {{ gitea_api_token.json.sha1 }}
             body: "{{ item.body }}"
           loop:
+            - organization: mc
+              body:
+                name: GitOps.ClusterAPI
+                # auto_init: true
+                # default_branch: main
+                description: ClusterAPI manifests
             - organization: mc
               body:
                 name: GitOps.Config
@@ -115,20 +121,26 @@
                 description: GitOps manifests
             - organization: wl
               body:
-                name: Template.GitOps.Config
+                name: GitOps.Config
                 # auto_init: true
                 # default_branch: main
                 description: GitOps manifests
+            - organization: wl
+              body:
+                name: GitOps.HelmCharts
+                # auto_init: true
+                # default_branch: main
+                description: Helm charts
           loop_control:
-            label: "{{ item.organization + '/' + item.body.name }}"
+            label: "{{ item.organization ~ '/' ~ item.body.name }}"
 
-        - name: Rebase/Push source gitops repository
-          ansible.builtin.shell:
-            cmd: |
-              git config --local http.sslVerify false
-              git remote set-url origin https://administrator:{{ vapp['metacluster.password'] | urlencode }}@git.{{ vapp['metacluster.fqdn'] }}/mc/GitOps.Config.git
-              git push
-            chdir: /opt/metacluster/git-repositories/gitops
+        # - name: Rebase/Push source gitops repository
+        #   ansible.builtin.shell:
+        #     cmd: |
+        #       git config --local http.sslVerify false
+        #       git remote set-url origin https://administrator:{{ vapp['metacluster.password'] | urlencode }}@git.{{ vapp['metacluster.fqdn'] }}/mc/GitOps.Config.git
+        #       git push
+        #     chdir: /opt/metacluster/git-repositories/gitops
 
       when: (gitea_existing_config.json is undefined) or (gitea_existing_config.json.data | length == 0)
 

ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/gitops.yml

@@ -5,10 +5,10 @@
         name: argo-cd
         chart_ref: /opt/metacluster/helm-charts/argo-cd
         release_namespace: argo-cd
-        create_namespace: yes
-        wait: no
+        create_namespace: true
+        wait: false
         kubeconfig: "{{ kubeconfig.path }}"
-        values: "{{ components.argocd.chart_values }}"
+        values: "{{ components['argo-cd'].chart_values }}"
 
     - name: Ensure argo-cd API availability
       ansible.builtin.uri:
@@ -18,7 +18,7 @@
       until:
         - api_readycheck.json.Version is defined
       retries: "{{ playbook.retries }}"
-      delay: "{{ playbook.delays.long }}"
+      delay: "{{ (storage_benchmark | int) * (playbook.delay.long | int) }}"
 
     - name: Generate argo-cd API token
       ansible.builtin.uri:
@@ -39,24 +39,29 @@
         mode: 0600
       vars:
         _template:
-          name: argocd-gitrepo-metacluster
+          name: gitrepo-mc-gitopsconfig
           namespace: argo-cd
-          uid: "{{ lookup('ansible.builtin.password', '/dev/null length=5 chars=ascii_lowercase,digits seed=inventory_hostname') }}"
-          privatekey: "{{ lookup('ansible.builtin.file', '~/.ssh/git_rsa_id') | indent(4, true) }}"
+          url: https://git.{{ vapp['metacluster.fqdn'] }}/mc/GitOps.Config.git
       notify:
         - Apply manifests
 
     - name: Create applicationset
       ansible.builtin.template:
         src: applicationset.j2
-        dest: /var/lib/rancher/k3s/server/manifests/{{ _template.name }}-manifest.yaml
+        dest: /var/lib/rancher/k3s/server/manifests/{{ _template.application.name }}-manifest.yaml
         owner: root
         group: root
         mode: 0600
       vars:
         _template:
-          name: argocd-applicationset-metacluster
-          namespace: argo-cd
+          application:
+            name: applicationset-metacluster
+            namespace: argo-cd
+          cluster:
+            url: https://kubernetes.default.svc
+          repository:
+            url: https://git.{{ vapp['metacluster.fqdn'] }}/mc/GitOps.Config.git
+            revision: main
       notify:
         - Apply manifests
 

ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/ingress.yml

@@ -1,3 +1,21 @@
+- name: Reconfigure traefik container for persistence
+  ansible.builtin.blockinfile:
+    path: /var/lib/rancher/k3s/server/manifests/traefik-config.yaml
+    block: |2
+          deployment:
+            initContainers:
+              - name: volume-permissions
+                image: busybox:1
+                command: ["sh", "-c", "touch /data/acme.json && chmod -Rv 600 /data/* && chown 65532:65532 /data/acme.json"]
+                volumeMounts:
+                  - name: data
+                    mountPath: /data
+          persistence:
+            enabled: true
+    marker: '    # {mark} ANSIBLE MANAGED BLOCK [persistence]'
+  notify:
+    - Apply manifests
+
 - name: Configure traefik dashboard ingress
   ansible.builtin.template:
     src: ingressroute.j2

ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/init.yml

@@ -1,7 +1,7 @@
 - name: Configure fallback name resolution
   ansible.builtin.lineinfile:
     path: /etc/hosts
-    line: "{{ vapp['guestinfo.ipaddress'] }}  {{ item + '.' + vapp['metacluster.fqdn'] }}"
+    line: "{{ vapp['guestinfo.ipaddress'] }}  {{ item ~ '.' ~ vapp['metacluster.fqdn'] }}"
     state: present
   loop:
     # TODO: Make this list dynamic
@@ -11,3 +11,73 @@
     - ingress
     - registry
     - storage
+
+- name: Initialize tempfile
+  ansible.builtin.tempfile:
+    state: file
+  register: stepca_password
+
+- name: Store password in tempfile
+  ansible.builtin.copy:
+    dest: "{{ stepca_password.path }}"
+    content: "{{ vapp['metacluster.password'] }}"
+  no_log: true
+
+- name: Generate step-ca helm chart values (including root CA certificate)
+  ansible.builtin.shell:
+    cmd: >-
+      step ca init \
+        --helm \
+        --deployment-type=standalone \
+        --name=ca.{{ vapp['metacluster.fqdn'] }} \
+        --dns=ca.{{ vapp['metacluster.fqdn'] }} \
+        --dns=step-certificates.step-ca.svc.cluster.local \
+        --dns=127.0.0.1 \
+        --address=:9000 \
+        --provisioner=admin \
+        --acme \
+        --password-file={{ stepca_password.path }}
+  register: stepca_values
+
+- name: Cleanup tempfile
+  ansible.builtin.file:
+    path: "{{ stepca_password.path }}"
+    state: absent
+  when: stepca_password.path is defined
+
+- name: Store root CA certificate
+  ansible.builtin.copy:
+    dest: /usr/local/share/ca-certificates/root_ca.crt
+    content: "{{ (stepca_values.stdout | from_yaml).inject.certificates.root_ca }}"
+
+- name: Update certificate truststore
+  ansible.builtin.command:
+    cmd: update-ca-certificates
+
+- name: Get all stored fully qualified container image names
+  ansible.builtin.shell:
+    cmd: >-
+      skopeo list-tags \
+        --insecure-policy \
+        docker-archive:./{{ item | basename }} | \
+      jq -r '.Tags[0]'
+    chdir: /opt/metacluster/container-images
+  register: registry_artifacts
+  loop: "{{ query('ansible.builtin.fileglob', '/opt/metacluster/container-images/*.tar') | sort }}"
+  loop_control:
+    label: "{{ item | basename }}"
+
+- name: Get source registries of all artifacts
+  ansible.builtin.set_fact:
+    source_registries: "{{ (source_registries | default([]) + [(item | split('/'))[0]]) | unique | sort }}"
+  loop: "{{ registry_artifacts | json_query('results[*].stdout') | select | sort }}"
+
+- name: Configure K3s node for private registry
+  ansible.builtin.template:
+    dest: /etc/rancher/k3s/registries.yaml
+    src: registries.j2
+  vars:
+    _template:
+      registries: "{{ source_registries }}"
+      hv:
+        fqdn: "{{ vapp['metacluster.fqdn'] }}"

ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/k3s.yml

@@ -7,6 +7,7 @@
       content: |
         kubelet-arg:
           - "config=/etc/rancher/k3s/kubelet.config"
+          - "image-gc-high-threshold=95"
     - filename: /etc/rancher/k3s/kubelet.config
       content: |
         apiVersion: kubelet.config.k8s.io/v1beta1
@@ -30,17 +31,6 @@
     INSTALL_K3S_EXEC: "server --cluster-init --token {{ vapp['metacluster.token'] | trim }} --tls-san {{ vapp['metacluster.vip'] }} --disable local-storage --config /etc/rancher/k3s/config.yaml"
   when: ansible_facts.services['k3s.service'] is undefined
 
-- name: Debug possible taints on k3s node
-  ansible.builtin.shell:
-    cmd: >-
-      while true;
-      do
-        kubectl get nodes -o custom-columns=NAME:.metadata.name,TAINTS:.spec.taints --no-headers | awk '{print strftime("%H:%M:%S"),$0;fflush();}' >> /var/log/taintlog
-        sleep 1
-      done
-  async: 1800
-  poll: 0
-
 - name: Ensure API availability
   ansible.builtin.uri:
     url: https://{{ vapp['guestinfo.ipaddress'] }}:6443/livez?verbose
@@ -50,7 +40,7 @@
   register: api_readycheck
   until: api_readycheck.json.apiVersion is defined
   retries: "{{ playbook.retries }}"
-  delay: "{{ playbook.delays.medium }}"
+  delay: "{{ (storage_benchmark | int) * (playbook.delay.medium | int) }}"
 
 - name: Install kubectl tab-completion
   ansible.builtin.shell:
@@ -72,3 +62,19 @@
     content: "{{ kubectl_config.stdout }}"
     mode: 0600
   no_log: true
+
+- name: Add label to node object
+  kubernetes.core.k8s:
+    name: "{{ ansible_facts.nodename | lower }}"
+    kind: Node
+    state: patched
+    definition:
+      metadata:
+        labels:
+          ova.airgappedk8s/moref_id: "{{ moref_id }}"
+    kubeconfig: "{{ kubeconfig.path }}"
+  register: k8snode_patch
+  until:
+    - k8snode_patch.result.metadata.labels['ova.airgappedk8s/moref_id'] is defined
+  retries: "{{ playbook.retries }}"
+  delay: "{{ (storage_benchmark | int) * (playbook.delay.medium | int) }}"

ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/registry.yml

@@ -5,10 +5,10 @@
         name: harbor
         chart_ref: /opt/metacluster/helm-charts/harbor
         release_namespace: harbor
-        create_namespace: yes
-        wait: no
+        create_namespace: true
+        wait: false
         kubeconfig: "{{ kubeconfig.path }}"
-        values: "{{ components.harbor.chart_values }}"
+        values: "{{ components['harbor'].chart_values }}"
 
     - name: Ensure harbor API availability
       ansible.builtin.uri:
@@ -19,7 +19,7 @@
         - api_readycheck.json.status is defined
         - api_readycheck.json.status == 'healthy'
       retries: "{{ playbook.retries }}"
-      delay: "{{ playbook.delays.long }}"
+      delay: "{{ (storage_benchmark | int) * (playbook.delay.long | int) }}"
 
     - name: Push images to registry
       ansible.builtin.shell:
@@ -40,30 +40,9 @@
       loop_control:
         label: "{{ item | basename }}"
       retries: "{{ playbook.retries }}"
-      delay: "{{ playbook.delays.short }}"
+      delay: "{{ ((storage_benchmark | float) * playbook.delay.short) | int }}"
       until: push_result is not failed
 
-    - name: Get all stored container images (=artifacts)
-      ansible.builtin.uri:
-        url: https://registry.{{ vapp['metacluster.fqdn'] }}/api/v2.0/search?q=library
-        method: GET
-      register: registry_artifacts
-
-    - name: Get source registries of all artifacts
-      ansible.builtin.set_fact:
-        source_registries: "{{ (source_registries | default([]) + [(item | split('/'))[1]]) | unique | sort }}"
-      loop: "{{ registry_artifacts.json.repository | json_query('[*].repository_name') }}"
-
-    - name: Configure K3s node for private registry
-      ansible.builtin.template:
-        dest: /etc/rancher/k3s/registries.yaml
-        src: registries.j2
-      vars:
-        _template:
-          data: "{{ source_registries }}"
-          hv:
-            fqdn: "{{ vapp['metacluster.fqdn'] }}"
-
   module_defaults:
     ansible.builtin.uri:
       validate_certs: no

ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/storage.yml

@@ -4,10 +4,10 @@
         name: longhorn
         chart_ref: /opt/metacluster/helm-charts/longhorn
         release_namespace: longhorn-system
-        create_namespace: yes
-        wait: no
+        create_namespace: true
+        wait: false
         kubeconfig: "{{ kubeconfig.path }}"
-        values: "{{ components.longhorn.chart_values }}"
+        values: "{{ components['longhorn'].chart_values }}"
 
     - name: Ensure longhorn API availability
       ansible.builtin.uri:
@@ -17,7 +17,7 @@
       until:
         - api_readycheck is not failed
       retries: "{{ playbook.retries }}"
-      delay: "{{ playbook.delays.long }}"
+      delay: "{{ (storage_benchmark | int) * (playbook.delay.long | int) }}"
 
   module_defaults:
     ansible.builtin.uri:

ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/preflight/tasks/vapp.yml

@@ -0,0 +1,25 @@
+- name: Check for expected vApp properties
+  ansible.builtin.assert:
+    that:
+      - vapp[item] is defined
+      - (vapp[item] | length) > 0
+    quiet: true
+  loop:
+    - deployment.type
+    - guestinfo.dnsserver
+    - guestinfo.gateway
+    - guestinfo.hostname
+    - guestinfo.ipaddress
+    - guestinfo.prefixlength
+    - guestinfo.rootsshkey
+    - hv.fqdn
+    - hv.password
+    - hv.username
+    - ippool.endip
+    - ippool.startip
+    - metacluster.fqdn
+    - metacluster.password
+    - metacluster.token
+    - metacluster.vip
+    - workloadcluster.name
+    - workloadcluster.vip

ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/clusterapi.yml

@@ -1,4 +1,7 @@
 - block:
+  # Below tasks circumvent usernames with `<domain>\<username>` format, which causes CAPV to
+  # incorrectly interpret the backslash (despite automatic escaping) as an escape sequence.
+  # `vcenter_session.user` will instead contain the username in `<username>@<domain>` format.
 
     - name: Generate vCenter API token
       ansible.builtin.uri:
@@ -13,7 +16,7 @@
         url: https://{{ vapp['hv.fqdn'] }}/api/session
         method: GET
         headers:
-          vmware-api-session-id: "{{ vcenter_api_token.json }}"
+          vmware-api-session-id: "{{ vcenterapi_token.json }}"
       register: vcenter_session
 
   module_defaults:
@@ -44,45 +47,32 @@
         resourcepool: "{{ vcenter_info.resourcepool }}"
         folder: "{{ vcenter_info.folder }}"
       cluster:
-        nodetemplate: "{{ (components.clusterapi.workload.node_template.url | basename | split('.'))[:-1] | join('.') }}"
+        nodetemplate: "{{ nodetemplate_inventorypath }}"
         publickey: "{{ vapp['guestinfo.rootsshkey'] }}"
         version: "{{ components.clusterapi.workload.version.k8s }}"
         vip: "{{ vapp['workloadcluster.vip'] }}"
 
-- name: Update image references to use local registry
-  ansible.builtin.replace:
-    dest: "{{ item.root + '/' + item.path }}"
-    regexp: '([ ]+image:[ "]+)(?!({{ _template.pattern }}|"{{ _template.pattern }}))'
-    replace: '\1{{ _template.pattern }}'
-  vars:
-    _template:
-      pattern: registry.{{ vapp['metacluster.fqdn'] }}/library/
-  loop: "{{ lookup('community.general.filetree', '/opt/metacluster/cluster-api') }}"
-  loop_control:
-    label: "{{ item.path }}"
-  when:
-    - item.path is search('.yaml')
-    - item.path is not search("clusterctl.yaml|metadata.yaml")
-
-- name: Generate kustomization template
+- name: Generate cluster-template kustomization manifest
   ansible.builtin.template:
     src: kustomization.cluster-template.j2
     dest: /opt/metacluster/cluster-api/infrastructure-vsphere/{{ components.clusterapi.management.version.infrastructure_vsphere }}/kustomization.yaml
   vars:
     _template:
-      fqdn: "{{ vapp['metacluster.fqdn'] }}"
+      network:
+        fqdn: "{{ vapp['metacluster.fqdn'] }}"
+        dnsserver: "{{ vapp['guestinfo.dnsserver'] }}"
+      nodesize:
+        cpu: "{{ config.clusterapi.size_matrix[ vapp['workloadcluster.nodesize'] ].cpu }}"
+        memory: "{{ config.clusterapi.size_matrix[ vapp['workloadcluster.nodesize'] ].memory }}"
       rootca: "{{ stepca_cm_certs.resources[0].data['root_ca.crt'] }}"
-      script:
-        # Base64 encoded; to avoid variable substitution when clusterctl parses the cluster-template.yml
-        encoded: IyEvYmluL2Jhc2gKdm10b29sc2QgLS1jbWQgJ2luZm8tZ2V0IGd1ZXN0aW5mby5vdmZFbnYnID4gL3RtcC9vdmZlbnYKCklQQWRkcmVzcz0kKHNlZCAtbiAncy8uKlByb3BlcnR5IG9lOmtleT0iZ3Vlc3RpbmZvLmludGVyZmFjZS4wLmlwLjAuYWRkcmVzcyIgb2U6dmFsdWU9IlwoW14iXSpcKS4qL1wxL3AnIC90bXAvb3ZmZW52KQpTdWJuZXRNYXNrPSQoc2VkIC1uICdzLy4qUHJvcGVydHkgb2U6a2V5PSJndWVzdGluZm8uaW50ZXJmYWNlLjAuaXAuMC5uZXRtYXNrIiBvZTp2YWx1ZT0iXChbXiJdKlwpLiovXDEvcCcgL3RtcC9vdmZlbnYpCkdhdGV3YXk9JChzZWQgLW4gJ3MvLipQcm9wZXJ0eSBvZTprZXk9Imd1ZXN0aW5mby5pbnRlcmZhY2UuMC5yb3V0ZS4wLmdhdGV3YXkiIG9lOnZhbHVlPSJcKFteIl0qXCkuKi9cMS9wJyAvdG1wL292ZmVudikKRE5TPSQoc2VkIC1uICdzLy4qUHJvcGVydHkgb2U6a2V5PSJndWVzdGluZm8uZG5zLnNlcnZlcnMiIG9lOnZhbHVlPSJcKFteIl0qXCkuKi9cMS9wJyAvdG1wL292ZmVudikKTUFDQWRkcmVzcz0kKHNlZCAtbiAncy8uKnZlOkFkYXB0ZXIgdmU6bWFjPSJcKFteIl0qXCkuKi9cMS9wJyAvdG1wL292ZmVudikKCm1hc2syY2lkcigpIHsKICBjPTAKICB4PTAkKCBwcmludGYgJyVvJyAkezEvLy4vIH0gKQoKICB3aGlsZSBbICR4IC1ndCAwIF07IGRvCiAgICBsZXQgYys9JCgoeCUyKSkgJ3g+Pj0xJwogIGRvbmUKCiAgZWNobyAkYwp9CgpQcmVmaXg9JChtYXNrMmNpZHIgJFN1Ym5ldE1hc2spCgpjYXQgPiAvZXRjL25ldHBsYW4vMDEtbmV0Y2ZnLnlhbWwgPDxFT0YKbmV0d29yazoKICB2ZXJzaW9uOiAyCiAgcmVuZGVyZXI6IG5ldHdvcmtkCiAgZXRoZXJuZXRzOgogICAgaWQwOgogICAgICBzZXQtbmFtZTogZXRoMAogICAgICBtYXRjaDoKICAgICAgICBtYWNhZGRyZXNzOiAkTUFDQWRkcmVzcwogICAgICBhZGRyZXNzZXM6CiAgICAgICAgLSAkSVBBZGRyZXNzLyRQcmVmaXgKICAgICAgZ2F0ZXdheTQ6ICRHYXRld2F5CiAgICAgIG5hbWVzZXJ2ZXJzOgogICAgICAgIGFkZHJlc3NlcyA6IFskRE5TXQpFT0YKcm0gL2V0Yy9uZXRwbGFuLzUwKi55YW1sIC1mCgpzdWRvIG5ldHBsYW4gYXBwbHk=
       runcmds:
         - update-ca-certificates
-        - bash /root/network.sh
+      registries: "{{ source_registries }}"
 
 - name: Store custom cluster-template
   ansible.builtin.copy:
     dest: /opt/metacluster/cluster-api/custom-cluster-template.yaml
-    content: "{{ lookup('kubernetes.core.kustomize', dir='/opt/metacluster/cluster-api/infrastructure-vsphere/' + components.clusterapi.management.version.infrastructure_vsphere ) }}"
+    content: "{{ lookup('kubernetes.core.kustomize', dir='/opt/metacluster/cluster-api/infrastructure-vsphere/' ~ components.clusterapi.management.version.infrastructure_vsphere ) }}"
 
 - name: Initialize Cluster API management cluster
   ansible.builtin.shell:
@@ -95,7 +85,7 @@
         --kubeconfig {{ kubeconfig.path }}
     chdir: /opt/metacluster/cluster-api
 
-- name: Ensure CAPI/CAPV controller availability
+- name: Ensure controller availability
   kubernetes.core.k8s_info:
     kind: Deployment
     name: "{{ item.name }}"
@@ -103,6 +93,8 @@
     wait: true
     kubeconfig: "{{ kubeconfig.path }}"
   loop:
+    - name: caip-in-cluster-controller-manager
+      namespace: caip-in-cluster-system
     - name: capi-controller-manager
       namespace: capi-system
     - name: capv-controller-manager
@@ -115,7 +107,8 @@
     clustersize: >-
       {{ {
         'controlplane': vapp['deployment.type'] | regex_findall('^cp(\d)+') | first,
-        'workers': vapp['deployment.type'] | regex_findall('w(\d)+$') | first
+        'worker': vapp['deployment.type'] | regex_findall('w(\d)+') | first,
+        'workerstorage': vapp['deployment.type'] | regex_findall('ws(\d)+$') | first
       } }}
 
 - name: Generate workload cluster manifest
@@ -124,25 +117,131 @@
       clusterctl generate cluster \
         {{ vapp['workloadcluster.name'] | lower }} \
         --control-plane-machine-count {{ clustersize.controlplane }} \
-        --worker-machine-count {{ clustersize.workers }} \
+        --worker-machine-count {{ clustersize.worker }} \
         --from ./custom-cluster-template.yaml \
         --config ./clusterctl.yaml \
         --kubeconfig {{ kubeconfig.path }}
     chdir: /opt/metacluster/cluster-api
   register: clusterctl_newcluster
 
-# TODO: move to git repo
+- name: Initialize tempfolder
+  ansible.builtin.tempfile:
+    state: directory
+  register: capi_clustermanifest
+
 - name: Save workload cluster manifest
   ansible.builtin.copy:
-    dest: /opt/metacluster/cluster-api/new-cluster.yaml
+    dest: "{{ capi_clustermanifest.path }}/new-cluster.yaml"
     content: "{{ clusterctl_newcluster.stdout }}"
-- name: Apply workload cluster manifest
-  kubernetes.core.k8s:
-    definition: >-
-      {{ clusterctl_newcluster.stdout }}
-    wait: yes
-    kubeconfig: "{{ kubeconfig.path }}"
-# TODO: move to git repo
+
+- name: Split manifest into separate files
+  ansible.builtin.shell:
+    cmd: >-
+      kubectl slice \
+        -f {{ capi_clustermanifest.path }}/new-cluster.yaml \
+        -o {{ capi_clustermanifest.path }}/manifests
+
+- name: Generate nodepool kustomization manifest
+  ansible.builtin.template:
+    src: kustomization.nodepool.j2
+    dest: "{{ capi_clustermanifest.path }}/kustomization.yaml"
+  vars:
+    _template:
+      cluster:
+        name: "{{ vapp['workloadcluster.name'] }}"
+      nodepool:
+        size: "{{ clustersize.workerstorage }}"
+        additionaldisk: "{{ vapp['workloadcluster.additionaldisk'] }}"
+
+- name: Store nodepool manifest
+  ansible.builtin.copy:
+    dest: "{{ capi_clustermanifest.path }}/manifests/nodepool-worker-storage.yaml"
+    content: "{{ lookup('kubernetes.core.kustomize', dir=capi_clustermanifest.path) }}"
+
+- name: Create in-cluster IpPool
+  ansible.builtin.template:
+    src: ippool.j2
+    dest: "{{ capi_clustermanifest.path }}/manifests/inclusterippool-{{ _template.cluster.name }}.yml"
+  vars:
+    _template:
+      cluster:
+        name: "{{ vapp['workloadcluster.name'] | lower }}"
+        namespace: default
+        network:
+          startip: "{{ vapp['ippool.startip'] }}"
+          endip: "{{ vapp['ippool.endip'] }}"
+          prefix: "{{ vapp['guestinfo.prefixlength'] }}"
+          gateway: "{{ vapp['guestinfo.gateway'] }}"
+
+- name: Initialize/Push git repository
+  ansible.builtin.shell:
+    cmd: |
+      git init
+      git config --global user.email "administrator@{{ vapp['metacluster.fqdn'] }}"
+      git config --global user.name "administrator"
+      git checkout -b main
+      git add ./manifests
+      git commit -m "Upload manifests"
+      git remote add origin https://git.{{ vapp['metacluster.fqdn'] }}/mc/GitOps.ClusterAPI.git
+      git push https://administrator:{{ vapp['metacluster.password'] | urlencode }}@git.{{ vapp['metacluster.fqdn'] }}/mc/GitOps.ClusterAPI.git --all
+    chdir: "{{ capi_clustermanifest.path }}"
+
+- name: Cleanup tempfolder
+  ansible.builtin.file:
+    path: "{{ capi_clustermanifest.path }}"
+    state: absent
+  when: capi_clustermanifest.path is defined
+
+- name: Configure Cluster API repository
+  ansible.builtin.template:
+    src: gitrepo.j2
+    dest: /var/lib/rancher/k3s/server/manifests/{{ _template.name }}-manifest.yaml
+    owner: root
+    group: root
+    mode: 0600
+  vars:
+    _template:
+      name: gitrepo-mc-gitopsclusterapi
+      namespace: argo-cd
+      url: https://git.{{ vapp['metacluster.fqdn'] }}/mc/GitOps.ClusterAPI.git
+  notify:
+    - Apply manifests
+
+- name: WORKAROUND - Wait for ingress ACME requests to complete
+  ansible.builtin.shell:
+    cmd: >-
+      openssl s_client -connect registry.{{ vapp['metacluster.fqdn'] }}:443 -servername registry.{{ vapp['metacluster.fqdn'] }} 2>/dev/null </dev/null | \
+        openssl x509 -noout -subject | \
+      grep 'subject=CN = registry.{{ vapp['metacluster.fqdn'] }}'
+  register: certificate_subject
+  until: certificate_subject is not failed
+  retries: "{{ playbook.retries }}"
+  delay: "{{ (storage_benchmark | int) * (playbook.delay.medium | int) }}"
+
+- name: Create application
+  ansible.builtin.template:
+    src: application.j2
+    dest: /var/lib/rancher/k3s/server/manifests/{{ _template.application.name }}-manifest.yaml
+    owner: root
+    group: root
+    mode: 0600
+  vars:
+    _template:
+      application:
+        name: application-clusterapi-workloadcluster
+        namespace: argo-cd
+      cluster:
+        name: https://kubernetes.default.svc
+        namespace: default
+      repository:
+        url: https://git.{{ vapp['metacluster.fqdn'] }}/mc/GitOps.ClusterAPI.git
+        path: manifests
+        revision: main
+  notify:
+    - Apply manifests
+
+- name: Trigger handlers
+  ansible.builtin.meta: flush_handlers
 
 - name: Wait for cluster to be available
   ansible.builtin.shell:
@@ -153,7 +252,7 @@
   register: cluster_readycheck
   until: cluster_readycheck is succeeded
   retries: "{{ playbook.retries }}"
-  delay: "{{ playbook.delays.long }}"
+  delay: "{{ (storage_benchmark | int) * (playbook.delay.long | int) }}"
 
 - name: Initialize tempfile
   ansible.builtin.tempfile:
@@ -180,6 +279,6 @@
   kubernetes.core.k8s:
     src: /opt/metacluster/cluster-api/cni-calico/{{ components.clusterapi.workload.version.calico }}/calico.yaml
     state: present
-    wait: yes
+    wait: true
     kubeconfig: "{{ capi_kubeconfig.path }}"
 # TODO: move to git repo

ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/gitops.yml

@@ -1,44 +1,105 @@
-- block:
+- name: Aggregate helm charts from filesystem
+  ansible.builtin.find:
+    path: /opt/workloadcluster/helm-charts
+    file_type: directory
+    recurse: false
+  register: helm_charts
 
-    - name: Generate service account in workload cluster
-      kubernetes.core.k8s:
-        template: serviceaccount.j2
-        state: present
+- name: Create hard-links to populate new git-repository
+  ansible.builtin.shell:
+    cmd: >-
+      cp -lr {{ item.path }}/ /opt/workloadcluster/git-repositories/gitops/charts
+  loop: "{{ helm_charts.files }}"
+  loop_control:
+    label: "{{ item.path | basename }}"
 
-    - name: Retrieve service account bearer token
-      kubernetes.core.k8s_info:
-        kind: ServiceAccount
-        name: "{{ _template.account.name }}"
-        namespace: "{{ _template.account.namespace }}"
-      register: workloadcluster_serviceaccount
+- name: Create subfolders
+  ansible.builtin.file:
+    path: /opt/workloadcluster/git-repositories/gitops/values/{{ item.key }}
+    state: directory
+  loop: "{{ query('ansible.builtin.dict', downstream_components) }}"
+  loop_control:
+    label: "{{ item.key }}"
 
-    - name: Retrieve service account bearer token
-      kubernetes.core.k8s_info:
-        kind: Secret
-        name: "{{ workloadcluster_serviceaccount.resources | json_query('[].secrets[].name') | first }}"
-        namespace: "{{ _template.account.namespace }}"
-      register: workloadcluster_bearertoken
+- name: Write chart values to file
+  ansible.builtin.copy:
+    dest: /opt/workloadcluster/git-repositories/gitops/values/{{ item.key }}/values.yaml
+    content: "{{ item.value.chart_values | default('# Empty') | to_nice_yaml(indent=2, width=4096) }}"
+  loop: "{{ query('ansible.builtin.dict', downstream_components) }}"
+  loop_control:
+    label: "{{ item.key }}"
 
-    - name: Register workload cluster in argo-cd
-      kubernetes.core.k8s:
-        template: cluster.j2
-        state: present
-        kubeconfig: "{{ kubeconfig.path }}"
-      vars:
-        _template:
-          cluster:
-            name: "{{ vapp['workloadcluster.name'] | lower }}"
-            secret: argocd-cluster-{{ vapp['workloadcluster.name'] | lower }}
-            url: https://{{ vapp['workloadcluster.vip'] }}:6443
-            token: "{{ workloadcluster_bearertoken.resources | json_query('[].data.token') }}"
+- name: Initialize/Push git repository
+  ansible.builtin.shell:
+    cmd: |
+      git init
+      git config --global user.email "administrator@{{ vapp['metacluster.fqdn'] }}"
+      git config --global user.name "administrator"
+      git checkout -b main
+      git add .
+      git commit -m "Upload charts"
+      git remote add origin https://git.{{ vapp['metacluster.fqdn'] }}/wl/GitOps.Config.git
+      git push https://administrator:{{ vapp['metacluster.password'] | urlencode }}@git.{{ vapp['metacluster.fqdn'] }}/wl/GitOps.Config.git --all
+    chdir: /opt/workloadcluster/git-repositories/gitops
 
+- name: Retrieve workload-cluster kubeconfig
+  kubernetes.core.k8s_info:
+    kind: Secret
+    name: "{{ vapp['workloadcluster.name'] }}-kubeconfig"
+    namespace: default
+    kubeconfig: "{{ kubeconfig.path }}"
+  register: secret_workloadcluster_kubeconfig
+
+- name: Register workload-cluster in argo-cd
+  kubernetes.core.k8s:
+    template: cluster.j2
+    state: present
+    kubeconfig: "{{ kubeconfig.path }}"
   vars:
     _template:
-      account:
-        name: argocd-sa
-        namespace: default
-      clusterrolebinding:
-        name: argocd-crb
-  module_defaults:
-    group/k8s:
-      kubeconfig: "{{ capi_kubeconfig.path }}"
+      cluster:
+        name: "{{ vapp['workloadcluster.name'] | lower }}"
+        secret: argocd-cluster-{{ vapp['workloadcluster.name'] | lower }}
+        url: https://{{ vapp['workloadcluster.vip'] }}:6443
+      kubeconfig:
+        ca: "{{ (secret_workloadcluster_kubeconfig.resources[0].data.value | b64decode | from_yaml).clusters[0].cluster['certificate-authority-data'] }}"
+        certificate: "{{ (secret_workloadcluster_kubeconfig.resources[0].data.value | b64decode | from_yaml).users[0].user['client-certificate-data'] }}"
+        key: "{{ (secret_workloadcluster_kubeconfig.resources[0].data.value | b64decode | from_yaml).users[0].user['client-key-data'] }}"
+
+- name: Configure workload-cluster GitOps repository
+  ansible.builtin.template:
+    src: gitrepo.j2
+    dest: /var/lib/rancher/k3s/server/manifests/{{ _template.name }}-manifest.yaml
+    owner: root
+    group: root
+    mode: 0600
+  vars:
+    _template:
+      name: gitrepo-wl-gitopsconfig
+      namespace: argo-cd
+      url: https://git.{{ vapp['metacluster.fqdn'] }}/wl/GitOps.Config.git
+  notify:
+    - Apply manifests
+
+- name: Create applicationset
+  ansible.builtin.template:
+    src: applicationset.j2
+    dest: /var/lib/rancher/k3s/server/manifests/{{ _template.application.name }}-manifest.yaml
+    owner: root
+    group: root
+    mode: 0600
+  vars:
+    _template:
+      application:
+        name: applicationset-workloadcluster
+        namespace: argo-cd
+      cluster:
+        url: https://{{ vapp['workloadcluster.vip'] }}:6443
+      repository:
+        url: https://git.{{ vapp['metacluster.fqdn'] }}/wl/GitOps.Config.git
+        revision: main
+  notify:
+    - Apply manifests
+
+- name: Trigger handlers
+  ansible.builtin.meta: flush_handlers

ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/hypervisor.yml

@@ -55,21 +55,3 @@
   loop: "{{ govc_inventory.results }}"
   loop_control:
     label: "{{ item.item.attribute }}"
-
-- name: Configure network protocol profile on hypervisor
-  ansible.builtin.shell:
-    cmd: >-
-      npp-prepper \
-        --server "{{ vapp['hv.fqdn'] }}" \
-        --username "{{ vapp['hv.username'] }}" \
-        --password "{{ vapp['hv.password'] }}" \
-        dc \
-        --name "{{ vcenter_info.datacenter }}" \
-        --portgroup "{{ vcenter_info.network }}" \
-        --startaddress {{ vapp['ippool.startip'] }} \
-        --endaddress {{ vapp['ippool.endip'] }} \
-        --netmask {{ (vapp['guestinfo.ipaddress'] + '/' + vapp['guestinfo.prefixlength']) | ansible.utils.ipaddr('netmask') }} \
-        {{ vapp['guestinfo.dnsserver'] | split(',') | map('trim') | map('regex_replace', '^', '--dnsserver ') | join(' ') }} \
-        --dnsdomain {{ vapp['metacluster.fqdn'] }} \
-        --gateway {{ vapp['guestinfo.gateway'] }} \
-        --force

ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/main.yml

@@ -1,5 +1,11 @@
 - import_tasks: hypervisor.yml
 - import_tasks: registry.yml
 - import_tasks: nodetemplates.yml
-- import_tasks: clusterapi.yml
-- import_tasks: gitops.yml
+
+- block:
+
+    - import_tasks: clusterapi.yml
+    - import_tasks: gitops.yml
+
+  when:
+    - vapp['deployment.type'] != 'core'

ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/nodetemplates.yml

@@ -1,81 +1,68 @@
 - block:
 
-    - name: Check for existing templates on hypervisor
+    - name: Check for existing template on hypervisor
       community.vmware.vmware_guest_info:
-        name: "{{ (item | basename | split('.'))[:-1] | join('.') }}"
+        name: "{{ (filename | basename | split('.'))[:-1] | join('.') }}"
       register: existing_ova
-      loop: "{{ query('ansible.builtin.fileglob', '/opt/workloadcluster/node-templates/*.ova') | sort }}"
       ignore_errors: yes
 
-    - name: Parse OVA files for network mappings
-      ansible.builtin.shell:
-        cmd: govc import.spec -json {{ item }}
-      environment:
-        GOVC_INSECURE: '1'
-        GOVC_URL: "{{ vapp['hv.fqdn'] }}"
-        GOVC_USERNAME: "{{ vapp['hv.username'] }}"
-        GOVC_PASSWORD: "{{ vapp['hv.password'] }}"
-      register: ova_spec
-      when: existing_ova.results[index] is failed
-      loop: "{{ query('ansible.builtin.fileglob', '/opt/workloadcluster/node-templates/*.ova') | sort }}"
-      loop_control:
-        index_var: index
+    - name: Store inventory path of existing template
+      ansible.builtin.set_fact:
+        nodetemplate_inventorypath: "{{ existing_ova.instance.hw_folder ~ '/' ~ existing_ova.instance.hw_name }}"
+      when: existing_ova is not failed
 
-    - name: Deploy OVA templates on hypervisor
-      community.vmware.vmware_deploy_ovf:
-        cluster: "{{ vcenter_info.cluster }}"
-        datastore: "{{ vcenter_info.datastore }}"
-        folder: "{{ vcenter_info.folder }}"
-        name: "{{ (item | basename | split('.'))[:-1] | join('.') }}"
-        networks: "{u'{{ ova_spec.results[index].stdout | from_json | json_query('NetworkMapping[0].Name') }}':u'{{ vcenter_info.network }}'}"
-        allow_duplicates: no
-        power_on: false
-        ovf: "{{ item }}"
-      register: ova_deploy
-      when: existing_ova.results[index] is failed
-      loop: "{{ query('ansible.builtin.fileglob', '/opt/workloadcluster/node-templates/*.ova') | sort }}"
-      loop_control:
-        index_var: index
+    - block:
 
-    - name: Add vApp properties on deployed VM's
-      ansible.builtin.shell:
-        cmd: >-
-          npp-prepper \
-            --server "{{ vapp['hv.fqdn'] }}" \
-            --username "{{ vapp['hv.username'] }}" \
-            --password "{{ vapp['hv.password'] }}" \
-            vm \
-            --datacenter "{{ vcenter_info.datacenter }}" \
-            --portgroup "{{ vcenter_info.network }}" \
-            --name "{{ item.instance.hw_name }}"
-      when: existing_ova.results[index] is failed
-      loop: "{{ ova_deploy.results }}"
-      loop_control:
-        index_var: index
-        label: "{{ item.item }}"
+        - name: Parse OVA file for network mappings
+          ansible.builtin.shell:
+            cmd: govc import.spec -json {{ filename }}
+          environment:
+            GOVC_INSECURE: '1'
+            GOVC_URL: "{{ vapp['hv.fqdn'] }}"
+            GOVC_USERNAME: "{{ vapp['hv.username'] }}"
+            GOVC_PASSWORD: "{{ vapp['hv.password'] }}"
+          register: ova_spec
 
-    - name: Create snapshot on deployed VM's
-      community.vmware.vmware_guest_snapshot:
-        folder: "{{ vcenter_info.folder }}"
-        name: "{{ item.instance.hw_name }}"
-        state: present
-        snapshot_name: "{{ ansible_date_time.iso8601_basic_short }}-base"
-      when: ova_deploy.results[index] is not skipped
-      loop: "{{ ova_deploy.results }}"
-      loop_control:
-        index_var: index
-        label: "{{ item.item }}"
+        - name: Deploy OVA template on hypervisor
+          community.vmware.vmware_deploy_ovf:
+            cluster: "{{ vcenter_info.cluster }}"
+            datastore: "{{ vcenter_info.datastore }}"
+            name: "{{ (filename | basename | split('.'))[:-1] | join('.') }}"
+            networks: "{u'{{ ova_spec.stdout | from_json | json_query('NetworkMapping[0].Name') }}':u'{{ vcenter_info.network }}'}"
+            allow_duplicates: no
+            power_on: false
+            ovf: "{{ filename }}"
+          register: ova_deploy
 
-    - name: Mark deployed VM's as templates
-      community.vmware.vmware_guest:
-        name: "{{ item.instance.hw_name }}"
-        is_template: yes
-      when: ova_deploy.results[index] is not skipped
-      loop: "{{ ova_deploy.results }}"
-      loop_control:
-        index_var: index
-        label: "{{ item.item }}"
+        - name: Add additional placeholder disk
+          community.vmware.vmware_guest_disk:
+            name: "{{ ova_deploy.instance.hw_name }}"
+            disk:
+              - size: 1Mb
+                scsi_controller: 1
+                scsi_type: paravirtual
+                unit_number: 0
 
+        # Disabled to allow disks to be resized; at the cost of cloning speed
+        # - name: Create snapshot on deployed VM
+        #   community.vmware.vmware_guest_snapshot:
+        #     name: "{{ ova_deploy.instance.hw_name }}"
+        #     state: present
+        #     snapshot_name: "{{ ansible_date_time.iso8601_basic_short }}-base"
+
+        - name: Mark deployed VM as templates
+          community.vmware.vmware_guest:
+            name: "{{ ova_deploy.instance.hw_name }}"
+            is_template: yes
+
+        - name: Store inventory path of deployed template
+          ansible.builtin.set_fact:
+            nodetemplate_inventorypath: "{{ ova_deploy.instance.hw_folder ~ '/' ~ ova_deploy.instance.hw_name }}"
+
+      when: existing_ova is failed
+
+  vars:
+    filename: "{{ query('ansible.builtin.fileglob', '/opt/workloadcluster/node-templates/*.ova') | first }}"
   module_defaults:
     group/vmware:
       hostname: "{{ vapp['hv.fqdn'] }}"
@@ -83,3 +70,4 @@
       username: "{{ vapp['hv.username'] }}"
       password: "{{ vapp['hv.password'] }}"
       datacenter: "{{ vcenter_info.datacenter }}"
+      folder: "{{ vcenter_info.folder }}"

ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/registry.yml

@@ -5,7 +5,7 @@
       url: https://registry.{{ vapp['metacluster.fqdn'] }}/api/v2.0/projects
       method: POST
       headers:
-        Authorization: "Basic {{ ('admin:' + vapp['metacluster.password']) | b64encode }}"
+        Authorization: "Basic {{ ('admin:' ~ vapp['metacluster.password']) | b64encode }}"
       body:
         project_name: kubeadm
         public: true
@@ -28,7 +28,7 @@
       url: https://registry.{{ vapp['metacluster.fqdn'] }}/api/v2.0/projects/kubeadm/repositories/{{ ( item | regex_findall('([^:/]+)') )[-2] }}/artifacts?from=library/{{ item | replace('/', '%2F') | replace(':', '%3A') }}
       method: POST
       headers:
-        Authorization: "Basic {{ ('admin:' + vapp['metacluster.password']) | b64encode }}"
+        Authorization: "Basic {{ ('admin:' ~ vapp['metacluster.password']) | b64encode }}"
       body:
         from: "{{ item }}"
     loop: "{{ kubeadm_images }}"

ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/application.j2

@@ -0,0 +1,16 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: {{ _template.application.name }}
+  namespace: {{ _template.application.namespace }}
+spec:
+  destination:
+    namespace: {{ _template.cluster.namespace }}
+    server: {{ _template.cluster.name }}
+  project: default
+  source:
+    repoURL: {{ _template.repository.url }}
+    path: {{ _template.repository.path }}
+    targetRevision: {{ _template.repository.revision }}
+  syncPolicy:
+    automated: {}

ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/applicationset.j2

@@ -1,28 +1,33 @@
 apiVersion: argoproj.io/v1alpha1
 kind: ApplicationSet
 metadata:
-  name: {{ _template.name }}
-  namespace: {{ _template.namespace }}
+  name: {{ _template.application.name }}
+  namespace: {{ _template.application.namespace }}
 spec:
   generators:
   - git:
-      repoURL: ssh://git@gitea-ssh.gitea.svc.cluster.local/mc/GitOps.Config.git
-      revision: HEAD
+      repoURL: {{ _template.repository.url }}
+      revision: {{ _template.repository.revision }}
       directories:
-      - path: metacluster-applicationset/*
+      - path: charts/*/*
   template:
     metadata:
-      name: {% raw %}'{{ path.basename }}'{% endraw +%}
+      name: application-{% raw %}{{ path.basename }}{% endraw +%}
     spec:
       project: default
       syncPolicy:
         automated:
           prune: true
           selfHeal: true
-      source:
-        repoURL: ssh://git@gitea-ssh.gitea.svc.cluster.local/mc/GitOps.Config.git
-        targetRevision: HEAD
+        syncOptions:
+        - CreateNamespace=true
+      sources:
+      - repoURL: {{ _template.repository.url }}
+        targetRevision: {{ _template.repository.revision }}
         path: {% raw %}'{{ path }}'{% endraw +%}
+        helm:
+          valueFiles:
+          - /values/{% raw %}{{ path.basename }}{% endraw %}/values.yaml
       destination:
-        server: https://kubernetes.default.svc
-        namespace: default
+        server: {{ _template.cluster.url }}
+        namespace: {% raw %}'{{ path[1] }}'{% endraw +%}

ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/cluster.j2

@@ -11,8 +11,10 @@ stringData:
   server: {{ _template.cluster.url }}
   config: |
     {
-      "bearerToken": "{{ _template.cluster.token }}",
       "tlsClientConfig": {
-        "insecure": true
+        "insecure": false,
+        "caData": "{{ _template.kubeconfig.ca }}",
+        "certData": "{{ _template.kubeconfig.certificate }}",
+        "keyData": "{{ _template.kubeconfig.key }}"
       }
     }

ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/gitrepo.j2

@@ -1,13 +1,9 @@
 apiVersion: v1
 kind: Secret
 metadata:
-  name: {{ _template.name }}-{{ _template.uid }}
+  name: {{ _template.name }}
   namespace: {{ _template.namespace }}
   labels:
     argocd.argoproj.io/secret-type: repository
 stringData:
-  url: ssh://git@gitea-ssh.gitea.svc.cluster.local/mc/GitOps.Config.git
-  name: {{ _template.name }}
-  insecure: 'true'
-  sshPrivateKey: |
-{{ _template.privatekey }}
+  url: {{ _template.url }}

ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/ippool.j2

@@ -0,0 +1,10 @@
+apiVersion: ipam.cluster.x-k8s.io/v1alpha1
+kind: InClusterIPPool
+metadata:
+  name: inclusterippool-{{ _template.cluster.name }}
+  namespace: {{ _template.cluster.namespace }}
+spec:
+  start: {{ _template.cluster.network.startip }}
+  end: {{ _template.cluster.network.endip }}
+  prefix: {{ _template.cluster.network.prefix }}
+  gateway: {{ _template.cluster.network.gateway }}

ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/kustomization.cluster-template.j2

@@ -3,8 +3,36 @@ kind: Kustomization
 resources:
 - cluster-template.yaml
 
-patchesStrategicMerge:
-  - |-
+patches:
+- patch: |-
+    apiVersion: v1
+    kind: Secret
+    metadata:
+      name: csi-vsphere-config
+      namespace: '${NAMESPACE}'
+    stringData:
+      data: |
+        apiVersion: v1
+        kind: Secret
+        metadata:
+          name: csi-vsphere-config
+          namespace: kube-system
+        stringData:
+          csi-vsphere.conf: |+
+            [Global]
+            insecure-flag = true
+            thumbprint = "${VSPHERE_TLS_THUMBPRINT}"
+            cluster-id = "${NAMESPACE}/${CLUSTER_NAME}"
+
+            [VirtualCenter "${VSPHERE_SERVER}"]
+            user = "${VSPHERE_USERNAME}"
+            password = "${VSPHERE_PASSWORD}"
+            datacenters = "${VSPHERE_DATACENTER}"
+
+            [Network]
+            public-network = "${VSPHERE_NETWORK}"
+        type: Opaque
+- patch: |-
     apiVersion: controlplane.cluster.x-k8s.io/v1beta1
     kind: KubeadmControlPlane
     metadata:
@@ -13,8 +41,8 @@ patchesStrategicMerge:
     spec:
       kubeadmConfigSpec:
         clusterConfiguration:
-          imageRepository: registry.{{ _template.fqdn }}/kubeadm
-  - |-
+          imageRepository: registry.{{ _template.network.fqdn }}/kubeadm
+- patch: |-
     apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
     kind: KubeadmConfigTemplate
     metadata:
@@ -24,8 +52,8 @@ patchesStrategicMerge:
       template:
         spec:
           clusterConfiguration:
-            imageRepository: registry.{{ _template.fqdn }}/kubeadm
-  - |-
+            imageRepository: registry.{{ _template.network.fqdn }}/kubeadm
+- patch: |-
     apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
     kind: KubeadmConfigTemplate
     metadata:
@@ -35,12 +63,21 @@ patchesStrategicMerge:
       template:
         spec:
           files:
-          - encoding: base64
-            content: |
-              {{ _template.script.encoded }}
-            permissions: '0744'
+          - content: |
+                [plugins."io.containerd.grpc.v1.cri".registry]
+                  config_path = "/etc/containerd/certs.d"
+            append: true
+            path: /etc/containerd/config.toml
+{% for registry in _template.registries %}
+          - content: |
+              server = "https://{{ registry }}"
+
+              [host."https://registry.{{ _template.network.fqdn }}/v2/library/{{ registry }}"]
+                capabilities = ["pull", "resolve"]
+                override_path = true
             owner: root:root
-            path: /root/network.sh
+            path: /etc/containerd/certs.d/{{ registry }}/hosts.toml
+{% endfor %}
           - content: |
               network: {config: disabled}
             owner: root:root
@@ -49,56 +86,170 @@ patchesStrategicMerge:
               {{ _template.rootca | indent(width=14, first=False) | trim }}
             owner: root:root
             path: /usr/local/share/ca-certificates/root_ca.crt
+- patch: |-
+    apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+    kind: VSphereMachineTemplate
+    metadata:
+      name: ${CLUSTER_NAME}
+      namespace: '${NAMESPACE}'
+    spec:
+      template:
+        spec:
+          network:
+            devices:
+            - dhcp4: false
+              addressesFromPools:
+              - apiGroup: ipam.cluster.x-k8s.io
+                kind: InClusterIPPool
+                name: inclusterippool-${CLUSTER_NAME}
+              nameservers:
+              - {{ _template.network.dnsserver }}
+              networkName: '${VSPHERE_NETWORK}'
+- patch: |-
+    apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+    kind: VSphereMachineTemplate
+    metadata:
+      name: ${CLUSTER_NAME}-worker
+      namespace: '${NAMESPACE}'
+    spec:
+      template:
+        spec:
+          network:
+            devices:
+            - dhcp4: false
+              addressesFromPools:
+              - apiGroup: ipam.cluster.x-k8s.io
+                kind: InClusterIPPool
+                name: inclusterippool-${CLUSTER_NAME}
+              nameservers:
+              - {{ _template.network.dnsserver }}
+              networkName: '${VSPHERE_NETWORK}'
 
-patchesJson6902:
-  - target:
-      group: controlplane.cluster.x-k8s.io
-      version: v1beta1
-      kind: KubeadmControlPlane
-      name: .*
-    patch: |-
-      - op: add
-        path: /spec/kubeadmConfigSpec/files/-
-        value:
-          encoding: base64
-          content: |
-            {{ _template.script.encoded }}
-          owner: root:root
-          path: /root/network.sh
-          permissions: '0744'
-      - op: add
-        path: /spec/kubeadmConfigSpec/files/-
-        value:
-          content: |
-            network: {config: disabled}
-          owner: root:root
-          path: /etc/cloud/cloud.cfg.d/99-disable-network-config.cfg
-      - op: add
-        path: /spec/kubeadmConfigSpec/files/-
-        value:
-          content: |
-            {{ _template.rootca | indent(width=12, first=False) | trim }}
-          owner: root:root
-          path: /usr/local/share/ca-certificates/root_ca.crt
-  - target:
-      group: bootstrap.cluster.x-k8s.io
-      version: v1beta1
-      kind: KubeadmConfigTemplate
-      name: .*
-    patch: |-
-{% for cmd in _template.runcmds %}
-      - op: add
-        path: /spec/template/spec/preKubeadmCommands/-
-        value: {{ cmd }}
+- target:
+    group: controlplane.cluster.x-k8s.io
+    version: v1beta1
+    kind: KubeadmControlPlane
+    name: .*
+  patch: |-
+    - op: add
+      path: /spec/kubeadmConfigSpec/files/-
+      value:
+        content: |
+            [plugins."io.containerd.grpc.v1.cri".registry]
+              config_path = "/etc/containerd/certs.d"
+        append: true
+        path: /etc/containerd/config.toml
+{% for registry in _template.registries %}
+    - op: add
+      path: /spec/kubeadmConfigSpec/files/-
+      value:
+        content: |
+          server = "https://{{ registry }}"
+
+          [host."https://registry.{{ _template.network.fqdn }}/v2/library/{{ registry }}"]
+            capabilities = ["pull", "resolve"]
+            override_path = true
+        owner: root:root
+        path: /etc/containerd/certs.d/{{ registry }}/hosts.toml
 {% endfor %}
-  - target:
-      group: controlplane.cluster.x-k8s.io
-      version: v1beta1
-      kind: KubeadmControlPlane
-      name: .*
-    patch: |-
+    - op: add
+      path: /spec/kubeadmConfigSpec/files/-
+      value:
+        content: |
+          network: {config: disabled}
+        owner: root:root
+        path: /etc/cloud/cloud.cfg.d/99-disable-network-config.cfg
+    - op: add
+      path: /spec/kubeadmConfigSpec/files/-
+      value:
+        content: |
+          {{ _template.rootca | indent(width=10, first=False) | trim }}
+        owner: root:root
+        path: /usr/local/share/ca-certificates/root_ca.crt
+- target:
+    group: bootstrap.cluster.x-k8s.io
+    version: v1beta1
+    kind: KubeadmConfigTemplate
+    name: .*
+  patch: |-
 {% for cmd in _template.runcmds %}
-      - op: add
-        path: /spec/kubeadmConfigSpec/preKubeadmCommands/-
-        value: {{ cmd }}
+    - op: add
+      path: /spec/template/spec/preKubeadmCommands/-
+      value: {{ cmd }}
 {% endfor %}
+- target:
+    group: controlplane.cluster.x-k8s.io
+    version: v1beta1
+    kind: KubeadmControlPlane
+    name: .*
+  patch: |-
+{% for cmd in _template.runcmds %}
+    - op: add
+      path: /spec/kubeadmConfigSpec/preKubeadmCommands/-
+      value: {{ cmd }}
+{% endfor %}
+
+- target:
+    group: infrastructure.cluster.x-k8s.io
+    version: v1beta1
+    kind: VSphereMachineTemplate
+    name: \${CLUSTER_NAME}
+  patch: |-
+    - op: replace
+      path: /metadata/name
+      value: ${CLUSTER_NAME}-master
+- target:
+    group: controlplane.cluster.x-k8s.io
+    version: v1beta1
+    kind: KubeadmControlPlane
+    name: \${CLUSTER_NAME}
+  patch: |-
+    - op: replace
+      path: /metadata/name
+      value: ${CLUSTER_NAME}-master
+    - op: replace
+      path: /spec/machineTemplate/infrastructureRef/name
+      value: ${CLUSTER_NAME}-master
+- target:
+    group: cluster.x-k8s.io
+    version: v1beta1
+    kind: Cluster
+    name: \${CLUSTER_NAME}
+  patch: |-
+    - op: replace
+      path: /spec/controlPlaneRef/name
+      value: ${CLUSTER_NAME}-master
+
+- target:
+    group: infrastructure.cluster.x-k8s.io
+    version: v1beta1
+    kind: VSphereMachineTemplate
+    name: \${CLUSTER_NAME}-worker
+  patch: |-
+    - op: replace
+      path: /spec/template/spec/numCPUs
+      value: {{ _template.nodesize.cpu }}
+    - op: replace
+      path: /spec/template/spec/memoryMiB
+      value: {{ _template.nodesize.memory }}
+- target:
+    group: cluster.x-k8s.io
+    version: v1beta1
+    kind: MachineDeployment
+    name: \${CLUSTER_NAME}-md-0
+  patch: |-
+    - op: replace
+      path: /metadata/name
+      value: ${CLUSTER_NAME}-worker
+    - op: replace
+      path: /spec/template/spec/bootstrap/configRef/name
+      value: ${CLUSTER_NAME}-worker
+- target:
+    group: bootstrap.cluster.x-k8s.io
+    version: v1beta1
+    kind: KubeadmConfigTemplate
+    name: \${CLUSTER_NAME}-md-0
+  patch: |-
+    - op: replace
+      path: /metadata/name
+      value: ${CLUSTER_NAME}-worker

ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/kustomization.nodepool.j2

@@ -0,0 +1,83 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- manifests/kubeadmconfigtemplate-{{ _template.cluster.name }}-worker.yaml
+- manifests/machinedeployment-{{ _template.cluster.name }}-worker.yaml
+- manifests/vspheremachinetemplate-{{ _template.cluster.name }}-worker.yaml
+
+patches:
+- patch: |-
+    apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
+    kind: KubeadmConfigTemplate
+    metadata:
+      name: {{ _template.cluster.name }}-worker
+      namespace: default
+    spec:
+      template:
+        spec:
+          diskSetup:
+            filesystems:
+            - device: /dev/sdb1
+              filesystem: ext4
+              label: blockstorage
+            partitions:
+            - device: /dev/sdb
+              layout: true
+              tableType: gpt
+          joinConfiguration:
+            nodeRegistration:
+              kubeletExtraArgs:
+                node-labels: "node.longhorn.io/create-default-disk=true"
+          mounts:
+          - - LABEL=blockstorage
+            - /mnt/blockstorage
+- patch: |-
+    apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+    kind: VSphereMachineTemplate
+    metadata:
+      name: {{ _template.cluster.name }}-worker
+      namespace: default
+    spec:
+      template:
+        spec:
+          additionalDisksGiB:
+          - {{ _template.nodepool.additionaldisk }}
+
+- target:
+    group: bootstrap.cluster.x-k8s.io
+    version: v1beta1
+    kind: KubeadmConfigTemplate
+    name: {{ _template.cluster.name }}-worker
+  patch: |-
+    - op: replace
+      path: /metadata/name
+      value: {{ _template.cluster.name }}-worker-storage
+
+- target:
+    group: cluster.x-k8s.io
+    version: v1beta1
+    kind: MachineDeployment
+    name: {{ _template.cluster.name }}-worker
+  patch: |-
+    - op: replace
+      path: /metadata/name
+      value: {{ _template.cluster.name }}-worker-storage
+    - op: replace
+      path: /spec/template/spec/bootstrap/configRef/name
+      value: {{ _template.cluster.name }}-worker-storage
+    - op: replace
+      path: /spec/template/spec/infrastructureRef/name
+      value: {{ _template.cluster.name }}-worker-storage
+    - op: replace
+      path: /spec/replicas
+      value: {{ _template.nodepool.size }}
+
+- target:
+    group: infrastructure.cluster.x-k8s.io
+    version: v1beta1
+    kind: VSphereMachineTemplate
+    name: {{ _template.cluster.name }}-worker
+  patch: |-
+    - op: replace
+      path: /metadata/name
+      value: {{ _template.cluster.name }}-worker-storage

ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/serviceaccount.j2

@@ -1,18 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: {{ _template.account.name }}
-  namespace: {{ _template.account.namespace }}
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: {{ _template.clusterrolebinding.name }}
-subjects:
-- kind: ServiceAccount
-  name: {{ _template.account.name }}
-  namespace: {{ _template.account.namespace }}
-roleRef:
-  kind: ClusterRole
-  name: cluster-admin
-  apiGroup: rbac.authorization.k8s.io

ansible/roles/firstboot/files/ansible_payload/common/roles/disks/tasks/main.yml

@@ -11,11 +11,27 @@
     lv: longhorn_lv
     size: 100%VG
 
+- name: Store begin timestamp
+  ansible.builtin.set_fact:
+    start_time: "{{ lookup('pipe', 'date +%s') }}"
+
 - name: Create filesystem
   community.general.filesystem:
     dev: /dev/mapper/longhorn_vg-longhorn_lv
     fstype: ext4
 
+- name: Store end timestamp
+  ansible.builtin.set_fact:
+    end_time: "{{ lookup('pipe', 'date +%s') }}"
+
+- name: Calculate crude storage benchmark
+  ansible.builtin.set_fact:
+    storage_benchmark: "{{ [storage_benchmark, (end_time | int - start_time | int)] | max }}"
+
+- name: Log benchmark actual duration
+  ansible.builtin.debug:
+    msg: "Benchmark actual duration: {{ (end_time | int - start_time | int) }} second(s)"
+
 - name: Mount dynamic disk
   ansible.posix.mount:
     path: /mnt/blockstorage

ansible/roles/firstboot/files/ansible_payload/common/roles/metacluster/tasks/assets.yml

@@ -8,5 +8,5 @@
     label: "{{ item | basename }}"
   # Probably should add a task before that ensures K3s node is fully initialized before starting imports; currently K3s goes away briefly during this loop
   retries: "{{ playbook.retries }}"
-  delay: "{{ playbook.delays.short }}"
+  delay: "{{ ((storage_benchmark | float) * playbook.delay.short) | int }}"
   until: import_result is not failed

ansible/roles/firstboot/files/ansible_payload/common/roles/preflight/tasks/main.yml

@@ -1 +1,2 @@
+- import_tasks: vapp.yml
 - import_tasks: vcenter.yml

ansible/roles/firstboot/files/ansible_payload/common/roles/preflight/tasks/vcenter.yml

@@ -5,7 +5,7 @@
         schema: vsphere
       register: vcenter_info
       retries: "{{ playbook.retries }}"
-      delay: "{{ playbook.delays.short }}"
+      delay: "{{ ((storage_benchmark | float) * playbook.delay.short) | int }}"
       until: vcenter_info is not failed
 
   module_defaults:

ansible/roles/firstboot/files/ansible_payload/common/roles/tty/tasks/main.yml

@@ -19,6 +19,25 @@
         executable: /opt/firstboot/tty.sh
         workingdir: /tmp/
       metacluster:
+        components:
+          - name: ArgoCD
+            url: https://gitops.${FQDN}
+            healthcheck: https://gitops.${FQDN}
+          - name: Gitea
+            url: https://git.${FQDN}
+            healthcheck: https://git.${FQDN}
+          - name: Harbor
+            url: https://registry.${FQDN}
+            healthcheck: https://registry.${FQDN}
+          - name: Longhorn
+            url: https://storage.${FQDN}
+            healthcheck: https://storage.${FQDN}
+          - name: StepCA
+            url: ''
+            healthcheck: https://ca.${FQDN}/health
+          - name: Traefik
+            url: https://ingress.${FQDN}
+            healthcheck: https://ingress.${FQDN}
         fqdn: "{{ vapp['metacluster.fqdn'] }}"
         vip: "{{ vapp['metacluster.vip'] }}"
   loop:

ansible/roles/firstboot/files/ansible_payload/common/roles/users/tasks/main.yml

@@ -25,7 +25,7 @@
     line: 'PasswordAuthentication yes'
     state: absent
   loop_control:
-    label: "{{ '[' + item.regex + '] ' + item.state }}"
+    label: "{{ '[' ~ item.regex ~ '] ' ~ item.state }}"
 
 - name: Create dedicated SSH keypair
   community.crypto.openssh_keypair:

ansible/roles/firstboot/files/ansible_payload/common/templates/registries.j2

@@ -1,8 +1,8 @@
 mirrors:
-{% for entry in _template.data %}
-  {{ entry }}:
+{% for registry in _template.registries %}
+  {{ registry }}:
     endpoint:
       - https://registry.{{ _template.hv.fqdn }}
     rewrite:
-      "(.*)": "library/{{ entry }}/$1"
+      "(.*)": "library/{{ registry }}/$1"
 {% endfor %}

ansible/roles/firstboot/files/ansible_payload/common/templates/tty.j2

@@ -12,12 +12,15 @@ DFLT='\033[0m'        # Reset colour
 LCLR='\033[K'         # Clear to end of line
 PRST='\033[0;0H'      # Reset cursor position
 
-# COMPONENTS=('ca' 'ingress' 'storage' 'registry' 'git' 'gitops')
-COMPONENTS=('storage' 'registry' 'git' 'gitops')
 FQDN='{{ _template.metacluster.fqdn }}'
 IPADDRESS='{{ _template.metacluster.vip }}'
 
-I=60
+declare -A COMPONENTS
+{% for component in _template.metacluster.components %}
+COMPONENTS["{{ component.name }}\t({{ component.url }})"]="{{ component.healthcheck }}"
+{% endfor %}
+
+I=0
 
 while /bin/true; do
   if [[ $I -gt 59 ]]; then
@@ -30,13 +33,13 @@ while /bin/true; do
   echo -e "${PRST}" > /dev/tty1
   echo -e "\n\n\t${DFLT}To manage this appliance, please connect to one of the following:${LCLR}\n" > /dev/tty1
 
-  for c in "${COMPONENTS[@]}"; do
-    STATUS=$(curl -ks "https://${c}.${FQDN}" -o /dev/null -w '%{http_code}')
+  for c in $( echo "${!COMPONENTS[@]}" | tr ' ' $'\n' | sort); do
+    STATUS=$(curl -kLs "${COMPONENTS[${c}]}" -o /dev/null -w '%{http_code}')
 
     if [[ "${STATUS}" -eq "200" ]]; then
-      echo -e "\t [${BGRN}+${DFLT}] ${BBLU}https://${c}.${FQDN}${DFLT}${LCLR}" > /dev/tty1
+      echo -e "\t [${BGRN}+${DFLT}] ${BBLU}${c}${DFLT}${LCLR}" > /dev/tty1
     else
-      echo -e "\t [${BRED}-${DFLT}] ${BBLU}https://${c}.${FQDN}${DFLT}${LCLR}" > /dev/tty1
+      echo -e "\t [${BRED}-${DFLT}] ${BBLU}${c}${DFLT}${LCLR}" > /dev/tty1
     fi
   done
 

ansible/roles/firstboot/files/ansible_payload/common/vars/defaults.yml

@@ -1,6 +1,23 @@
 playbook:
-  retries: 5
-  delays:
-    long: 60
-    medium: 30
-    short: 10
+  retries: 10
+  delay:
+    # These values are multiplied with the value of `storage_benchmark`
+    long: 2
+    medium: 1
+    short: 0.5
+
+# This default value is updated during the playbook, based on an I/O intensive operation
+storage_benchmark: 30
+
+config:
+  clusterapi:
+    size_matrix:
+      small:
+        cpu: 2
+        memory: 6144
+      medium:
+        cpu: 4
+        memory: 8192
+      large:
+        cpu: 8
+        memory: 16384

ansible/roles/firstboot/files/ansible_payload/upgrade/playbook.yml

@@ -13,7 +13,8 @@
     - users
     - disks
     - metacluster
-    # - workloadcluster
+    - workloadcluster
+    - decommission
     - tty
     - cleanup
   handlers:

ansible/roles/firstboot/files/ansible_payload/upgrade/roles/decommission/tasks/k3s.yml

@@ -0,0 +1,35 @@
+- name: Cordon node
+  kubernetes.core.k8s_drain:
+    name: "{{ decom_nodename }}"
+    state: cordon
+    kubeconfig: "{{ kubeconfig.path }}"
+
+- name: Drain node
+# Currently does not work; returns with error "Failed to delete pod [...] due to: Too Many Requests"
+# See also: https://github.com/ansible-collections/kubernetes.core/issues/474
+#   kubernetes.core.k8s_drain:
+#     name: "{{ decom_nodename }}"
+#     state: drain
+#     delete_options:
+#       ignore_daemonsets: true
+#       delete_emptydir_data: true
+#     kubeconfig: "{{ kubeconfig.path }}"
+  ansible.builtin.shell:
+    cmd: >-
+      kubectl drain {{ decom_nodename }} \
+        --delete-emptydir-data \
+        --ignore-daemonsets
+  register: nodedrain_results
+  until:
+    - nodedrain_results is not failed
+    - (nodedrain_results.stdout_lines | last) is match('node/.* drained')
+  retries: "{{ playbook.retries }}"
+  delay: "{{ ((storage_benchmark | float) * playbook.delay.short) | int }}"
+
+- name: Delete node
+  kubernetes.core.k8s:
+    name: "{{ decom_nodename }}"
+    kind: node
+    state: absent
+    wait: true
+    kubeconfig: "{{ kubeconfig.path }}"

ansible/roles/firstboot/files/ansible_payload/upgrade/roles/decommission/tasks/main.yml

@@ -0,0 +1,18 @@
+- name: Lookup node name and moref-id for decommissioning
+  ansible.builtin.set_fact:
+    decom_nodename: >-
+      {{
+        lookup('kubernetes.core.k8s', kind='Node', kubeconfig=(kubeconfig.path)) |
+          json_query('[? metadata.name != `' ~ ansible_facts.nodename ~ '`].metadata.name') |
+          first
+      }}
+    decom_morefid: >-
+      {{
+        lookup('kubernetes.core.k8s', kind='Node', kubeconfig=(kubeconfig.path)) |
+          json_query('[? metadata.name != `' ~ ansible_facts.nodename ~ '`].metadata.labels."ova.airgappedk8s/moref_id"') |
+          first
+      }}
+
+- import_tasks: storage.yml
+- import_tasks: k3s.yml
+- import_tasks: virtualmachine.yml

ansible/roles/firstboot/files/ansible_payload/upgrade/roles/decommission/tasks/storage.yml

@@ -0,0 +1,27 @@
+- name: Disable disk scheduling and evict replicas
+  kubernetes.core.k8s:
+    api_version: longhorn.io/v1beta2
+    kind: lhn
+    name: "{{ decom_nodename }}"
+    namespace: longhorn-system
+    state: patched
+    definition: |
+      spec:
+        allowScheduling: false
+        evictionRequested: true
+    kubeconfig: "{{ kubeconfig.path }}"
+
+- name: Reduce replica amount for each volume
+  kubernetes.core.k8s:
+    api_version: longhorn.io/v1beta2
+    kind: volume
+    name: "{{ item.metadata.name }}"
+    namespace: longhorn-system
+    state: patched
+    definition: |
+      spec:
+        numberOfReplicas: {{ (lookup('kubernetes.core.k8s', kind='node', kubeconfig=(kubeconfig.path)) | length | int) - 1 }}
+    kubeconfig: "{{ kubeconfig.path }}"
+  loop: "{{ query('kubernetes.core.k8s', api_version='longhorn.io/v1beta2', kind='volume', namespace='longhorn-system', kubeconfig=(kubeconfig.path)) }}"
+  loop_control:
+    label: "{{ item.metadata.name }}"

ansible/roles/firstboot/files/ansible_payload/upgrade/roles/decommission/tasks/virtualmachine.yml

@@ -0,0 +1,26 @@
+- block:
+
+    - name: Lookup VM name
+      community.vmware.vmware_guest_info:
+        moid: "{{ decom_morefid }}"
+      register: virtualmachine_details
+
+    - name: Power off VM
+      community.vmware.vmware_guest:
+        name: "{{ virtualmachine_details.instance.hw_name }}"
+        folder: "{{ virtualmachine_details.instance.hw_folder }}"
+        state: poweredoff
+
+    # - name: Delete VM
+    #   community.vmware.vmware_guest:
+    #     name: "{{ virtualmachine_details.hw_name }}"
+    #     folder: "{{ virtualmachine_details.hw_folder }}"
+    #     state: absent
+
+  module_defaults:
+    group/vmware:
+      hostname: "{{ vapp['hv.fqdn'] }}"
+      validate_certs: no
+      username: "{{ vapp['hv.username'] }}"
+      password: "{{ vapp['hv.password'] }}"
+      datacenter: "{{ vcenter_info.datacenter }}"

ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/certauthority.yml

@@ -0,0 +1,52 @@
+- block:
+
+    - name: Initialize tempfile
+      ansible.builtin.tempfile:
+        state: file
+      register: values_file
+
+    - name: Lookup current chart values
+      kubernetes.core.helm_info:
+        name: step-certificates
+        namespace: step-ca
+        kubeconfig: "{{ kubeconfig.path }}"
+      register: stepca_values
+
+    - name: Write chart values w/ password to tempfile
+      ansible.builtin.copy:
+        dest: "{{ values_file.path }}"
+        content: "{{ stepca_values.status | json_query('values') | to_yaml }}"
+      no_log: true
+
+    - name: Upgrade step-ca chart
+      kubernetes.core.helm:
+        name: step-certificates
+        chart_ref: /opt/metacluster/helm-charts/step-certificates
+        release_namespace: step-ca
+        wait: false
+        kubeconfig: "{{ kubeconfig.path }}"
+        values_files:
+          - "{{ values_file.path }}"
+
+    - name: Cleanup tempfile
+      ansible.builtin.file:
+        path: "{{ values_file.path }}"
+        state: absent
+      when: values_file.path is defined
+
+    - name: Ensure step-ca API availability
+      ansible.builtin.uri:
+        url: https://ca.{{ vapp['metacluster.fqdn'] }}/health
+        method: GET
+      register: api_readycheck
+      until:
+        - api_readycheck.json.status is defined
+        - api_readycheck.json.status == 'ok'
+      retries: "{{ playbook.retries }}"
+      delay: "{{ (storage_benchmark | int) * (playbook.delay.long | int) }}"
+
+  module_defaults:
+    ansible.builtin.uri:
+      validate_certs: no
+      status_code: [200, 201]
+      body_format: json

ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/containerimages.yml

@@ -0,0 +1,50 @@
+- block:
+
+    - name: Push images to registry
+      ansible.builtin.shell:
+        cmd: >-
+          skopeo copy \
+            --insecure-policy \
+            --dest-tls-verify=false \
+            --dest-creds admin:{{ vapp['metacluster.password'] }} \
+            docker-archive:./{{ item | basename }} \
+            docker://registry.{{ vapp['metacluster.fqdn'] }}/library/$( \
+              skopeo list-tags \
+                --insecure-policy \
+                docker-archive:./{{ item | basename }} | \
+              jq -r '.Tags[0]')
+        chdir: /opt/metacluster/container-images/
+      register: push_result
+      loop: "{{ query('ansible.builtin.fileglob', '/opt/metacluster/container-images/*.tar') | sort }}"
+      loop_control:
+        label: "{{ item | basename }}"
+      retries: "{{ playbook.retries }}"
+      delay: "{{ ((storage_benchmark | float) * playbook.delay.short) | int }}"
+      until: push_result is not failed
+
+    - name: Get all stored container images (=artifacts)
+      ansible.builtin.uri:
+        url: https://registry.{{ vapp['metacluster.fqdn'] }}/api/v2.0/search?q=library
+        method: GET
+      register: registry_artifacts
+
+    - name: Get source registries of all artifacts
+      ansible.builtin.set_fact:
+        source_registries: "{{ (source_registries | default([]) + [(item | split('/'))[1]]) | unique | sort }}"
+      loop: "{{ registry_artifacts.json.repository | json_query('[*].repository_name') }}"
+
+    - name: Configure K3s node for private registry
+      ansible.builtin.template:
+        dest: /etc/rancher/k3s/registries.yaml
+        src: registries.j2
+      vars:
+        _template:
+          data: "{{ source_registries }}"
+          hv:
+            fqdn: "{{ vapp['metacluster.fqdn'] }}"
+
+  module_defaults:
+    ansible.builtin.uri:
+      validate_certs: no
+      status_code: [200, 201, 401]
+      body_format: json

ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/git.yml

@@ -0,0 +1,27 @@
+- block:
+
+    - name: Upgrade gitea chart
+      kubernetes.core.helm:
+        name: gitea
+        chart_ref: /opt/metacluster/helm-charts/gitea
+        release_namespace: gitea
+        wait: false
+        kubeconfig: "{{ kubeconfig.path }}"
+        values: "{{ components['gitea'].chart_values }}"
+
+    - name: Ensure gitea API availability
+      ansible.builtin.uri:
+        url: https://git.{{ vapp['metacluster.fqdn'] }}/api/healthz
+        method: GET
+      register: api_readycheck
+      until:
+        - api_readycheck.json.status is defined
+        - api_readycheck.json.status == 'pass'
+      retries: "{{ playbook.retries }}"
+      delay: "{{ (storage_benchmark | int) * (playbook.delay.long | int) }}"
+
+  module_defaults:
+    ansible.builtin.uri:
+      validate_certs: no
+      status_code: [200, 201]
+      body_format: json

ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/gitops.yml

@@ -0,0 +1,26 @@
+- block:
+
+    - name: Upgrade argo-cd chart
+      kubernetes.core.helm:
+        name: argo-cd
+        chart_ref: /opt/metacluster/helm-charts/argo-cd
+        release_namespace: argo-cd
+        wait: false
+        kubeconfig: "{{ kubeconfig.path }}"
+        values: "{{ components['argo-cd'].chart_values }}"
+
+    - name: Ensure argo-cd API availability
+      ansible.builtin.uri:
+        url: https://gitops.{{ vapp['metacluster.fqdn'] }}/api/version
+        method: GET
+      register: api_readycheck
+      until:
+        - api_readycheck.json.Version is defined
+      retries: "{{ playbook.retries }}"
+      delay: "{{ (storage_benchmark | int) * (playbook.delay.long | int) }}"
+
+  module_defaults:
+    ansible.builtin.uri:
+      validate_certs: no
+      status_code: [200, 201]
+      body_format: json

ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/init.yml

@@ -1,7 +1,7 @@
 - name: Configure fallback name resolution
   ansible.builtin.lineinfile:
     path: /etc/hosts
-    line: "{{ vapp['metacluster.vip'] }}  {{ item + '.' + vapp['metacluster.fqdn'] }}"
+    line: "{{ vapp['metacluster.vip'] }}  {{ item ~ '.' ~ vapp['metacluster.fqdn'] }}"
     state: present
   loop:
     # TODO: Make this list dynamic
@@ -28,3 +28,8 @@
 - name: Update certificate truststore
   ansible.builtin.command:
     cmd: update-ca-certificates
+
+- name: Remove redundant files
+  ansible.builtin.file:
+    path: /var/lib/rancher/k3s/server/manifests/traefik-config.yaml
+    state: absent

ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/k3s.yml

@@ -27,7 +27,8 @@
     chdir: /opt/metacluster/k3s
   environment:
     INSTALL_K3S_SKIP_DOWNLOAD: 'true'
-    INSTALL_K3S_EXEC: "server --token {{ vapp['metacluster.token'] | trim }} --server https://{{ vapp['metacluster.vip'] }}:6443 --disable local-storage --config /etc/rancher/k3s/config.yaml"
+    # To prevent from overwriting traefik's existing configuration, "disable" it on this new node
+    INSTALL_K3S_EXEC: "server --token {{ vapp['metacluster.token'] | trim }} --server https://{{ vapp['metacluster.vip'] }}:6443 --disable local-storage,traefik --config /etc/rancher/k3s/config.yaml"
   when: ansible_facts.services['k3s.service'] is undefined
 
 - name: Ensure API availability
@@ -39,7 +40,7 @@
   register: api_readycheck
   until: api_readycheck.json.apiVersion is defined
   retries: "{{ playbook.retries }}"
-  delay: "{{ playbook.delays.medium }}"
+  delay: "{{ (storage_benchmark | int) * (playbook.delay.medium | int) }}"
 
 - name: Install kubectl tab-completion
   ansible.builtin.shell:
@@ -61,3 +62,19 @@
     content: "{{ kubectl_config.stdout }}"
     mode: 0600
   no_log: true
+
+- name: Add label to node object
+  kubernetes.core.k8s:
+    name: "{{ ansible_facts.nodename | lower }}"
+    kind: Node
+    state: patched
+    definition:
+      metadata:
+        labels:
+          ova.airgappedk8s/moref_id: "{{ moref_id }}"
+    kubeconfig: "{{ kubeconfig.path }}"
+  register: k8snode_patch
+  until:
+    - k8snode_patch.result.metadata.labels['ova.airgappedk8s/moref_id'] is defined
+  retries: "{{ playbook.retries }}"
+  delay: "{{ (storage_benchmark | int) * (playbook.delay.medium | int) }}"

ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/main.yml

@@ -1,9 +1,9 @@
 - import_tasks: init.yml
-- import_tasks: registry.yml
+- import_tasks: containerimages.yml
 - import_tasks: k3s.yml
 - import_tasks: assets.yml
-# - import_tasks: ingress.yml
 - import_tasks: storage.yml
-# - import_tasks: certauthority.yml
-# - import_tasks: git.yml
-# - import_tasks: gitops.yml
+- import_tasks: registry.yml
+- import_tasks: certauthority.yml
+- import_tasks: git.yml
+- import_tasks: gitops.yml

ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/registry.yml

@@ -1,47 +1,24 @@
 - block:
 
-    - name: Push images to registry
-      ansible.builtin.shell:
-        cmd: >-
-          skopeo copy \
-            --insecure-policy \
-            --dest-tls-verify=false \
-            --dest-creds admin:{{ vapp['metacluster.password'] }} \
-            docker-archive:./{{ item | basename }} \
-            docker://registry.{{ vapp['metacluster.fqdn'] }}/library/$( \
-              skopeo list-tags \
-                --insecure-policy \
-                docker-archive:./{{ item | basename }} | \
-              jq -r '.Tags[0]')
-        chdir: /opt/metacluster/container-images/
-      register: push_result
-      loop: "{{ query('ansible.builtin.fileglob', '/opt/metacluster/container-images/*.tar') | sort }}"
-      loop_control:
-        label: "{{ item | basename }}"
-      retries: "{{ playbook.retries }}"
-      delay: "{{ playbook.delays.short }}"
-      until: push_result is not failed
+    - name: Upgrade harbor chart
+      kubernetes.core.helm:
+        name: harbor
+        chart_ref: /opt/metacluster/helm-charts/harbor
+        release_namespace: harbor
+        wait: false
+        kubeconfig: "{{ kubeconfig.path }}"
+        values: "{{ components['harbor'].chart_values }}"
 
-    - name: Get all stored container images (=artifacts)
+    - name: Ensure harbor API availability
       ansible.builtin.uri:
-        url: https://registry.{{ vapp['metacluster.fqdn'] }}/api/v2.0/search?q=library
+        url: https://registry.{{ vapp['metacluster.fqdn'] }}/api/v2.0/health
         method: GET
-      register: registry_artifacts
-
-    - name: Get source registries of all artifacts
-      ansible.builtin.set_fact:
-        source_registries: "{{ (source_registries | default([]) + [(item | split('/'))[1]]) | unique | sort }}"
-      loop: "{{ registry_artifacts.json.repository | json_query('[*].repository_name') }}"
-
-    - name: Configure K3s node for private registry
-      ansible.builtin.template:
-        dest: /etc/rancher/k3s/registries.yaml
-        src: registries.j2
-      vars:
-        _template:
-          data: "{{ source_registries }}"
-          hv:
-            fqdn: "{{ vapp['metacluster.fqdn'] }}"
+      register: api_readycheck
+      until:
+        - api_readycheck.json.status is defined
+        - api_readycheck.json.status == 'healthy'
+      retries: "{{ playbook.retries }}"
+      delay: "{{ (storage_benchmark | int) * (playbook.delay.long | int) }}"
 
   module_defaults:
     ansible.builtin.uri:

ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/storage.yml

@@ -1,14 +1,53 @@
-- name: Increase replicas for each volume
-  kubernetes.core.k8s:
-    api_version: longhorn.io/v1beta2
-    kind: volume
-    name: "{{ item.metadata.name }}"
-    namespace: longhorn-system
-    state: patched
-    definition: |
-      spec:
-        numberOfReplicas: {{ lookup('kubernetes.core.k8s', kind='node', kubeconfig=(kubeconfig.path)) | length | int }}
-    kubeconfig: "{{ kubeconfig.path }}"
-  loop: "{{ lookup('kubernetes.core.k8s', api_version='longhorn.io/v1beta2', kind='volume', namespace='longhorn-system', kubeconfig=(kubeconfig.path)) }}"
-  loop_control:
-    label: "{{ item.metadata.name }}"
+- block:
+
+    - name: Increase replicas for each volume
+      kubernetes.core.k8s:
+        api_version: longhorn.io/v1beta2
+        kind: volume
+        name: "{{ item.metadata.name }}"
+        namespace: longhorn-system
+        state: patched
+        definition: |
+          spec:
+            numberOfReplicas: {{ lookup('kubernetes.core.k8s', kind='node', kubeconfig=(kubeconfig.path)) | length | int }}
+        kubeconfig: "{{ kubeconfig.path }}"
+      loop: "{{ query('kubernetes.core.k8s', api_version='longhorn.io/v1beta2', kind='volume', namespace='longhorn-system', kubeconfig=(kubeconfig.path)) }}"
+      loop_control:
+        label: "{{ item.metadata.name }}"
+
+    - name: Wait for replica rebuilds to complete
+      ansible.builtin.uri:
+        url: https://storage.{{ vapp['metacluster.fqdn'] }}/v1/volumes
+        method: GET
+      register: volume_details
+      until:
+        - volume_details.json is defined
+        - (volume_details.json | json_query('data[? state==`attached`].robustness') | unique | length) == 1
+        - (volume_details.json | json_query('data[? state==`attached`].robustness') | first) == "healthy"
+      retries: "{{ ( playbook.retries * 2) | int }}"
+      delay: "{{ (storage_benchmark | int) * (playbook.delay.long | int) }}"
+
+    - name: Upgrade longhorn chart
+      kubernetes.core.helm:
+        name: longhorn
+        chart_ref: /opt/metacluster/helm-charts/longhorn
+        release_namespace: longhorn-system
+        wait: false
+        kubeconfig: "{{ kubeconfig.path }}"
+        values: "{{ components['longhorn'].chart_values }}"
+
+    - name: Ensure longhorn API availability
+      ansible.builtin.uri:
+        url: https://storage.{{ vapp['metacluster.fqdn'] }}/v1
+        method: GET
+      register: api_readycheck
+      until:
+        - api_readycheck is not failed
+      retries: "{{ playbook.retries }}"
+      delay: "{{ (storage_benchmark | int) * (playbook.delay.long | int) }}"
+
+  module_defaults:
+    ansible.builtin.uri:
+      validate_certs: no
+      status_code: [200, 201]
+      body_format: json

ansible/roles/firstboot/files/ansible_payload/upgrade/roles/preflight/tasks/main.yml

@@ -1,2 +1,3 @@
+- import_tasks: vapp.yml
 - import_tasks: vcenter.yml
 - import_tasks: metacluster.yml

ansible/roles/firstboot/files/ansible_payload/upgrade/roles/preflight/tasks/metacluster.yml

@@ -4,3 +4,8 @@
     method: GET
     validate_certs: no
     status_code: [200, 401]
+  register: api_readycheck
+  until:
+    - api_readycheck.json.apiVersion is defined
+  retries: "{{ playbook.retries }}"
+  delay: "{{ (storage_benchmark | int) * (playbook.delay.medium | int) }}"

ansible/roles/firstboot/files/ansible_payload/upgrade/roles/preflight/tasks/vapp.yml

@@ -0,0 +1,20 @@
+- name: Check for expected vApp properties
+  ansible.builtin.assert:
+    that:
+      - vapp[item] is defined
+      - (vapp[item] | length) > 0
+    quiet: true
+  loop:
+    - guestinfo.dnsserver
+    - guestinfo.gateway
+    - guestinfo.hostname
+    - guestinfo.ipaddress
+    - guestinfo.prefixlength
+    - guestinfo.rootsshkey
+    - hv.fqdn
+    - hv.password
+    - hv.username
+    - metacluster.fqdn
+    - metacluster.password
+    - metacluster.token
+    - metacluster.vip

ansible/roles/firstboot/files/ansible_payload/upgrade/roles/workloadcluster/tasks/clusterapi.yml

@@ -0,0 +1 @@
+# ...

ansible/roles/firstboot/files/ansible_payload/upgrade/roles/workloadcluster/tasks/hypervisor.yml

@@ -0,0 +1,57 @@
+- name: Gather hypervisor details
+  ansible.builtin.shell:
+    cmd: govc ls -L {{ item.moref }} | awk -F/ '{print ${{ item.part }}}'
+  environment:
+    GOVC_INSECURE: '1'
+    GOVC_URL: "{{ vapp['hv.fqdn'] }}"
+    GOVC_USERNAME: "{{ vapp['hv.username'] }}"
+    GOVC_PASSWORD: "{{ vapp['hv.password'] }}"
+  register: govc_inventory
+  loop:
+    - attribute: cluster
+      moref: >-
+        $(govc object.collect -json VirtualMachine:{{ moref_id }} | \
+          jq -r '.[] | select(.Name == "runtime").Val.Host | .Type + ":" + .Value')
+      part: (NF-1)
+    - attribute: datacenter
+      moref: VirtualMachine:{{ moref_id }}
+      part: 2
+    - attribute: datastore
+      moref: >-
+        $(govc object.collect -json VirtualMachine:{{ moref_id }} | \
+          jq -r '.[] | select(.Name == "datastore").Val.ManagedObjectReference | .[].Type + ":" + .[].Value')
+      part: NF
+    - attribute: folder
+      moref: >-
+        $(govc object.collect -json VirtualMachine:{{ moref_id }} | \
+          jq -r '.[] | select(.Name == "parent").Val | .Type + ":" + .Value')
+      part: 0
+    # - attribute: host
+    #   moref: >-
+    #     $(govc object.collect -json VirtualMachine:{{ moref_id }} | \
+    #       jq -r '.[] | select(.Name == "runtime").Val.Host | .Type + ":" + .Value')
+    #   part: NF
+    - attribute: network
+      moref: >-
+        $(govc object.collect -json VirtualMachine:{{ moref_id }} | \
+          jq -r '.[] | select(.Name == "network").Val.ManagedObjectReference | .[].Type + ":" + .[].Value')
+      part: NF
+    - attribute: resourcepool
+      moref: >-
+        $(govc object.collect -json VirtualMachine:{{ moref_id }} | \
+          jq -r '.[] | select(.Name == "resourcePool").Val | .Type + ":" + .Value')
+      part: 0
+  loop_control:
+    label: "{{ item.attribute }}"
+
+- name: Retrieve hypervisor TLS thumbprint
+  ansible.builtin.shell:
+    cmd: openssl s_client -connect {{ vapp['hv.fqdn'] }}:443 < /dev/null 2>/dev/null | openssl x509 -fingerprint -noout -in /dev/stdin | awk -F'=' '{print $2}'
+  register: tls_thumbprint
+
+- name: Store hypervisor details in dictionary
+  ansible.builtin.set_fact:
+    vcenter_info: "{{ vcenter_info | default({}) | combine({ item.item.attribute : item.stdout }) }}"
+  loop: "{{ govc_inventory.results }}"
+  loop_control:
+    label: "{{ item.item.attribute }}"

ansible/roles/firstboot/files/ansible_payload/upgrade/roles/workloadcluster/tasks/main.yml

@@ -0,0 +1,4 @@
+- import_tasks: hypervisor.yml
+- import_tasks: registry.yml
+- import_tasks: nodetemplates.yml
+# - import_tasks: clusterapi.yml

ansible/roles/firstboot/files/ansible_payload/upgrade/roles/workloadcluster/tasks/nodetemplates.yml

@@ -0,0 +1,73 @@
+- block:
+
+    - name: Check for existing template on hypervisor
+      community.vmware.vmware_guest_info:
+        name: "{{ (filename | basename | split('.'))[:-1] | join('.') }}"
+      register: existing_ova
+      ignore_errors: yes
+
+    - name: Store inventory path of existing template
+      ansible.builtin.set_fact:
+        nodetemplate_inventorypath: "{{ existing_ova.instance.hw_folder ~ '/' ~ existing_ova.instance.hw_name }}"
+      when: existing_ova is not failed
+
+    - block:
+
+        - name: Parse OVA file for network mappings
+          ansible.builtin.shell:
+            cmd: govc import.spec -json {{ filename }}
+          environment:
+            GOVC_INSECURE: '1'
+            GOVC_URL: "{{ vapp['hv.fqdn'] }}"
+            GOVC_USERNAME: "{{ vapp['hv.username'] }}"
+            GOVC_PASSWORD: "{{ vapp['hv.password'] }}"
+          register: ova_spec
+
+        - name: Deploy OVA template on hypervisor
+          community.vmware.vmware_deploy_ovf:
+            cluster: "{{ vcenter_info.cluster }}"
+            datastore: "{{ vcenter_info.datastore }}"
+            name: "{{ (filename | basename | split('.'))[:-1] | join('.') }}"
+            networks: "{u'{{ ova_spec.stdout | from_json | json_query('NetworkMapping[0].Name') }}':u'{{ vcenter_info.network }}'}"
+            allow_duplicates: no
+            power_on: false
+            ovf: "{{ filename }}"
+          register: ova_deploy
+
+        - name: Add additional placeholder disk
+          community.vmware.vmware_guest_disk:
+            name: "{{ ova_deploy.instance.hw_name }}"
+            disk:
+              - size: 1Gb
+                scsi_controller: 1
+                scsi_type: paravirtual
+                unit_number: 0
+
+        # Disabled to allow disks to be resized; at the cost of cloning speed
+        # - name: Create snapshot on deployed VM
+        #   community.vmware.vmware_guest_snapshot:
+        #     name: "{{ ova_deploy.instance.hw_name }}"
+        #     state: present
+        #     snapshot_name: "{{ ansible_date_time.iso8601_basic_short }}-base"
+
+        - name: Mark deployed VM as templates
+          community.vmware.vmware_guest:
+            name: "{{ ova_deploy.instance.hw_name }}"
+            is_template: yes
+
+        - name: Store inventory path of deployed template
+          ansible.builtin.set_fact:
+            nodetemplate_inventorypath: "{{ ova_deploy.instance.hw_folder ~ '/' ~ ova_deploy.instance.hw_name }}"
+
+      when: existing_ova is failed
+
+  vars:
+    filename: "{{ query('ansible.builtin.fileglob', '/opt/metacluster/node-templates/*.ova') | first }}"
+  module_defaults:
+    group/vmware:
+      hostname: "{{ vapp['hv.fqdn'] }}"
+      validate_certs: no
+      username: "{{ vapp['hv.username'] }}"
+      password: "{{ vapp['hv.password'] }}"
+      datacenter: "{{ vcenter_info.datacenter }}"
+      folder: "{{ vcenter_info.folder }}"

ansible/roles/firstboot/files/ansible_payload/upgrade/roles/workloadcluster/tasks/registry.yml

@@ -0,0 +1,17 @@
+- block:
+
+  - name: Copy kubeadm container images to dedicated project
+    ansible.builtin.uri:
+      url: https://registry.{{ vapp['metacluster.fqdn'] }}/api/v2.0/projects/kubeadm/repositories/{{ ( item | regex_findall('([^:/]+)') )[-2] }}/artifacts?from=library/{{ item | replace('/', '%2F') | replace(':', '%3A') }}
+      method: POST
+      headers:
+        Authorization: "Basic {{ ('admin:' ~ vapp['metacluster.password']) | b64encode }}"
+      body:
+        from: "{{ item }}"
+    loop: "{{ lookup('ansible.builtin.file', '/opt/metacluster/cluster-api/imagelist').splitlines() }}"
+
+  module_defaults:
+    ansible.builtin.uri:
+      validate_certs: no
+      status_code: [200, 201, 409]
+      body_format: json

ansible/roles/firstboot/templates/firstboot.j2

@@ -1,4 +1,10 @@
 #!/bin/bash
 
+# Workaround for ansible output regression
+export PYTHONUNBUFFERED=1
+
 # Apply firstboot configuration w/ ansible
-/usr/local/bin/ansible-playbook -e "PYTHONUNBUFFERED=1" /opt/firstboot/ansible/playbook.yml | tee -a /var/log/firstboot.log > /dev/tty1 2>&1
+/usr/local/bin/ansible-playbook /opt/firstboot/ansible/playbook.yml | tee -a /var/log/firstboot.log > /dev/tty1 2>&1
+
+# Cleanup console
+clear > /dev/tty1

ansible/roles/os/tasks/main.yml

@@ -1,17 +1,7 @@
-- name: Disable tty logins
-  import_tasks: tty.yml
-
-- name: Remove snapd
-  import_tasks: snapd.yml
-
-- name: Remove cloud-init
-  import_tasks: cloud-init.yml
-
-- name: Configure default logging
-  import_tasks: logging.yml
-
-- name: Configure services
-  import_tasks: services.yml
-
-- name: Install packages
-  import_tasks: packages.yml
+- import_tasks: tty.yml
+- import_tasks: snapd.yml
+- import_tasks: cloud-init.yml
+- import_tasks: logging.yml
+- import_tasks: services.yml
+- import_tasks: packages.yml
+- import_tasks: sysctl.yml

ansible/roles/os/tasks/sysctl.yml

@@ -0,0 +1,11 @@
+- name: Configure inotify limits
+  ansible.posix.sysctl:
+    name: "{{ item.name }}"
+    value: "{{ item.value }}"
+  loop:
+    - name: fs.inotify.max_user_instances
+      value: '512'
+    - name: fs.inotify.max_user_watches
+      value: '524288'
+  loop_control:
+    label: "{{ item.name ~ '=' ~ item.value }}"

ansible/vars/metacluster.yml

@@ -1,13 +1,7 @@
 platform:
 
   k3s:
-    version: v1.26.0+k3s1
-
-  gitops:
-    repository:
-      uri: https://code.spamasaurus.com/djpbessems/GitOps.MetaCluster.git
-      # revision: v0.1.0
-      revision: HEAD
+    version: v1.25.9+k3s1
 
   packaged_components:
     - name: traefik
@@ -19,20 +13,10 @@ platform:
               - "--certificatesResolvers.stepca.acme.storage=/data/acme.json"
               - "--certificatesResolvers.stepca.acme.tlsChallenge=true"
               - "--certificatesresolvers.stepca.acme.certificatesduration=24"
-            deployment:
-              initContainers:
-                - name: volume-permissions
-                  image: busybox:1
-                  command: ["sh", "-c", "touch /data/acme.json && chmod -Rv 600 /data/* && chown 65532:65532 /data/acme.json"]
-                  volumeMounts:
-                    - name: data
-                      mountPath: /data
             globalArguments: []
             ingressRoute:
               dashboard:
                 enabled: false
-            persistence:
-              enabled: true
             ports:
               ssh:
                 port: 8022
@@ -42,10 +26,19 @@ platform:
               websecure:
                 tls:
                   certResolver: stepca
+            updateStrategy:
+              type: Recreate
+              rollingUpdate: null
 
   helm_repositories:
     - name: argo
       url: https://argoproj.github.io/argo-helm
+    - name: authentik
+      url: https://charts.goauthentik.io
+    # - name: codecentric
+    #   url: https://codecentric.github.io/helm-charts
+    # - name: dex
+    #   url: https://charts.dexidp.io
     - name: gitea-charts
       url: https://dl.gitea.io/charts/
     - name: harbor
@@ -54,6 +47,8 @@ platform:
       url: https://charts.jetstack.io
     - name: longhorn
       url: https://charts.longhorn.io
+    - name: prometheus-community
+      url: https://prometheus-community.github.io/helm-charts
     - name: smallstep
       url: https://smallstep.github.io/helm-charts/
 
@@ -61,8 +56,7 @@ components:
 
   argo-cd:
     helm:
-      # version: 4.9.7  # (= ArgoCD v2.4.2)
-      version: 5.14.1  # (= ArgoCD v2.5.2)
+      version: 5.27.4  # (= ArgoCD v2.6.7)
       chart: argo/argo-cd
       parse_logic: helm template . | yq --no-doc eval '.. | .image? | select(.)' | sort -u | awk '!/ /'
       chart_values: !unsafe |
@@ -77,9 +71,35 @@ components:
             hosts:
               - gitops.{{ vapp['metacluster.fqdn'] }}
 
+  authentik:
+    helm:
+      version: 2023.3.1
+      chart: authentik/authentik
+      parse_logic: helm template . --set postgresql.enabled=true,redis.enabled=true | yq --no-doc eval '.. | .image? | select(.)' | sort -u | awk '!/ /'
+      chart_values: !unsafe |
+        authentik:
+          avatars: none
+          secret_key: "{{ lookup('ansible.builtin.password', '/dev/null length=64 chars=ascii_lowercase,digits seed=' ~ vapp['guestinfo.hostname']) }}"
+          postgresql:
+            password: "{{ lookup('ansible.builtin.password', '/dev/null length=32 chars=ascii_lowercase,digits seed=' ~ vapp['guestinfo.hostname']) }}"
+        env:
+          AUTHENTIK_BOOTSTRAP_PASSWORD: "{{ vapp['metacluster.password'] }}"
+        ingress:
+          enabled: true
+          hosts:
+            - host: auth.{{ vapp['metacluster.fqdn'] }}
+              paths:
+                - path: "/"
+                  pathType: Prefix
+        postgresql:
+          enabled: true
+          postgresqlPassword: "{{ lookup('ansible.builtin.password', '/dev/null length=32 chars=ascii_lowercase,digits seed=' ~ vapp['guestinfo.hostname']) }}"
+        redis:
+          enabled: true
+
   cert-manager:
     helm:
-      version: 1.10.1
+      version: 1.11.0
       chart: jetstack/cert-manager
       parse_logic: helm template . | yq --no-doc eval '.. | .image? | select(.)' | sort -u | awk '!/ /'
       # chart_values: !unsafe |
@@ -89,24 +109,71 @@ components:
     management:
       version:
         # Must match the version referenced at `dependencies.static_binaries[.filename==clusterctl].url`
-        base: v1.3.2
+        base: v1.4.0
         # Must match the version referenced at `components.cert-manager.helm.version`
-        cert_manager: v1.10.1
-        infrastructure_vsphere: v1.5.1
-        ipam_incluster: v0.1.0-alpha.1
+        cert_manager: v1.11.0
+        infrastructure_vsphere: v1.6.0
+        ipam_incluster: v0.1.0-alpha.2
+        # Refer to `https://console.cloud.google.com/gcr/images/cloud-provider-vsphere/GLOBAL/cpi/release/manager` for available tags
+        cpi_vsphere: v1.25.2
     workload:
       version:
-        calico: v3.24.5
-        # k8s: v1.25.5
-        k8s: v1.23.5
+        calico: v3.25.0
+        k8s: v1.25.10
       node_template:
-        # Refer to `https://github.com/kubernetes-sigs/cluster-api-provider-vsphere/blob/v1.3.5/README.md#kubernetes-versions-with-published-ovas` for a list of supported node templates
-        # url: https://storage.googleapis.com/capv-templates/v1.25.5/ubuntu-2004-kube-v1.25.5.ova
-        url: https://storage.googleapis.com/capv-images/release/v1.23.5/ubuntu-2004-kube-v1.23.5.ova
+        url: https://{{ repo_username }}:{{ repo_password }}@sn.itch.fyi/Repository/rel/ubuntu-2204-kube-v1.25.10.ova
+
+  # dex:
+  #   helm:
+  #     version: 0.13.0 # (= Dex 2.35.3)
+  #     chart: dex/dex
+  #     parse_logic: helm template . | yq --no-doc eval '.. | .image? | select(.)' | sort -u | awk '!/ /'
+  #     chart_values: !unsafe |
+  #       config:
+  #         connectors:
+  #           - type: ldap
+  #             id: ldap
+  #             name: "LDAP"
+  #             config:
+  #               host: "{{ vapp['ldap.fqdn'] }}:636"
+  #               insecureNoSSL: false
+  #               insecureSkipVerify: true
+  #               bindDN: "{{ vapp['ldap.dn'] }}"
+  #               bindPW: "{{ vapp['ldap.password'] }}"
+
+  #               usernamePrompt: "Username"
+  #               userSearch:
+  #                 baseDN: OU=Administrators,OU=Useraccounts,DC=bessems,DC=eu
+  #                 filter: "(objectClass=person)"
+  #                 username: userPrincipalName
+  #                 idAttr: DN
+  #                 emailAttr: userPrincipalName
+  #                 nameAttr: cn
+
+  #               groupSearch:
+  #                 baseDN: OU=Roles,OU=Groups,DC=bessems,DC=eu
+  #                 filter: "(objectClass=group)"
+  #                 userMatchers:
+  #                 - userAttr: DN
+  #                   groupAttr: member
+  #                 nameAttr: cn
+  #         enablePasswordDB: true
+  #         issuer: https://oidc.{{ vapp['metacluster.fqdn'] }}
+  #         storage:
+  #           type: kubernetes
+  #           config:
+  #             inCluster: true
+  #       ingress:
+  #         enabled: true
+  #         hosts:
+  #           - host: oidc.{{ vapp['metacluster.fqdn'] }}
+  #             paths:
+  #               - path: /
+  #                 pathType: Prefix
 
   gitea:
     helm:
-      version: v6.0.3 # (= Gitea v1.17.3)
+      version: v7.0.2 # (= Gitea v1.18.3)
       chart: gitea-charts/gitea
       parse_logic: helm template . | yq --no-doc eval '.. | .image? | select(.)' | sort -u | sed '/:/!s/$/:latest/'
       chart_values: !unsafe |
@@ -137,7 +204,7 @@ components:
 
   harbor:
     helm:
-      version: 1.10.2  # (= Harbor v2.6.2)
+      version: 1.11.0  # (= Harbor v2.7.0)
       chart: harbor/harbor
       parse_logic: helm template . | yq --no-doc eval '.. | .image? | select(.)' | sort -u | awk '!/ /'
       chart_values: !unsafe |
@@ -158,13 +225,56 @@ components:
             registry:
               size: 25Gi
 
+  # keycloakx:
+  #   helm:
+  #     version: 2.1.1  # (= Keycloak 20.0.3)
+  #     chart: codecentric/keycloakx
+  #     parse_logic: helm template . | yq --no-doc eval '.. | .image? | select(.)' | sort -u | awk '!/ /'
+  #     chart_values: !unsafe |
+  #       command:
+  #         - "/opt/keycloak/bin/kc.sh"
+  #         - "start"
+  #         - "--http-enabled=true"
+  #         - "--http-port=8080"
+  #         - "--hostname-strict=false"
+  #         - "--hostname-strict-https=false"
+  #       extraEnv: |
+  #         - name: KEYCLOAK_ADMIN
+  #           value: admin
+  #         - name: KEYCLOAK_ADMIN_PASSWORD
+  #           value: {{ vapp['metacluster.password'] }}
+  #         - name: KC_PROXY
+  #           value: "passthrough"
+  #         - name: JAVA_OPTS_APPEND
+  #           value: >-
+  #             -Djgroups.dns.query={% raw %}{{ include "keycloak.fullname" . }}{% endraw %}-headless
+  #       ingress:
+  #         enabled: true
+  #         rules:
+  #           - host: keycloak.{{ vapp['metacluster.fqdn'] }}
+  #             paths:
+  #               - path: /
+  #                 pathType: Prefix
+  #         tls: []
+
+  kube-prometheus-stack:
+    helm:
+      version: 45.2.0
+      chart: prometheus-community/kube-prometheus-stack
+      parse_logic: helm template . | yq --no-doc eval '.. | .image? | select(.)' | sort -u | awk '!/ /'
+      chart_values: !unsafe |
+        alertmanager:
+          enabled: false
+        global:
+          imageRegistry: registry.{{ vapp['metacluster.fqdn'] }}
+
   kubevip:
     # Must match the version referenced at `dependencies.container_images`
     version: v0.5.8
 
   longhorn:
     helm:
-      version: 1.4.0
+      version: 1.4.1
       chart: longhorn/longhorn
       parse_logic: cat values.yaml | yq eval '.. | select(has("repository")) | .repository + ":" + .tag'
       chart_values: !unsafe |
@@ -180,20 +290,11 @@ components:
 
   step-certificates:
     helm:
-      # version: 1.18.2+20220324
       version: 1.23.0
       chart: smallstep/step-certificates
       parse_logic: helm template . | yq --no-doc eval '.. | .image? | select(.)' | sed '/:/!s/$/:latest/' | sort -u
       chart_values: !unsafe |
         ca:
-          bootstrap:
-            postInitHook: |
-              echo '{{ vapp["metacluster.password"] }}' > ~/pwfile
-              step ca provisioner add acme \
-                --type ACME \
-                --password-file=~/pwfile \
-                --force-cn
-              rm ~/pwfile
           dns: ca.{{ vapp['metacluster.fqdn'] }},step-certificates.step-ca.svc.cluster.local,127.0.0.1
           password: "{{ vapp['metacluster.password'] }}"
           provisioner:
@@ -233,7 +334,7 @@ dependencies:
 
   static_binaries:
     - filename: clusterctl
-      url: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.3.2/clusterctl-linux-amd64
+      url: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.4.0/clusterctl-linux-amd64
     - filename: govc
       url: https://github.com/vmware/govmomi/releases/download/v0.29.0/govc_Linux_x86_64.tar.gz
       archive: compressed
@@ -241,10 +342,11 @@ dependencies:
       url: https://get.helm.sh/helm-v3.10.2-linux-amd64.tar.gz
       archive: compressed
       extra_opts: --strip-components=1
-    - filename: npp-prepper
-      url: https://code.spamasaurus.com/api/packages/djpbessems/generic/npp-prepper/v0.4.5/npp-prepper
+    - filename: kubectl-slice
+      url: https://github.com/patrickdappollonio/kubectl-slice/releases/download/v1.2.5/kubectl-slice_linux_x86_64.tar.gz
+      archive: compressed
     - filename: skopeo
-      url: https://code.spamasaurus.com/api/packages/djpbessems/generic/skopeo/v1.11.0-dev/skopeo
+      url: https://code.spamasaurus.com/api/packages/djpbessems/generic/skopeo/v1.12.0/skopeo_linux_amd64
     - filename: step
       url: https://dl.step.sm/gh-release/cli/gh-release-header/v0.23.0/step_linux_0.23.0_amd64.tar.gz
       archive: compressed

ansible/vars/workloadcluster.yml

@@ -0,0 +1,27 @@
+downstream:
+
+  helm_repositories:
+    - name: longhorn
+      url: https://charts.longhorn.io
+    - name: sealed-secrets
+      url: https://bitnami-labs.github.io/sealed-secrets
+
+  helm_charts:
+
+    longhorn:
+      version: 1.4.1
+      chart: longhorn/longhorn
+      namespace: longhorn-system
+      parse_logic: cat values.yaml | yq eval '.. | select(has("repository")) | .repository + ":" + .tag'
+      chart_values: !unsafe |
+        defaultSettings:
+          createDefaultDiskLabeledNodes: true
+          defaultDataPath: /mnt/blockstorage
+
+    sealed-secrets:
+      version: 2.8.1  # (= Sealed Secrets v0.20.2)
+      chart: sealed-secrets/sealed-secrets
+      namespace: sealed-secrets
+      parse_logic: helm template . | yq --no-doc eval '.. | .image? | select(.)' | sort -u | awk '!/ /'
+      # chart_values: !unsafe |
+      #   # Empty

packer/build.pkr.hcl

@@ -6,16 +6,16 @@ packer {
 build {
   source "vsphere-iso.ubuntu" {
     name = "bootstrap"
-    vm_name = "${var.vm_name}-bootstrap"
+    vm_name = "bld_${var.vm_name}_bootstrap"
   }
 
   source "vsphere-iso.ubuntu" {
     name = "upgrade"
-    vm_name = "${var.vm_name}-upgrade"
+    vm_name = "bld_${var.vm_name}_upgrade"
   }
 
   provisioner "ansible" {
-    pause_before     = "2m30s"
+    pause_before     = "45s"
 
     playbook_file    = "ansible/playbook.yml"
     user             = "ubuntu"
@@ -24,11 +24,15 @@ build {
       "PYTHONUNBUFFERED=1"
     ]
     use_proxy        = "false"
+    collections_path = "ansible/collections"
+
     extra_arguments  = [
       "--extra-vars", "appliancetype=${source.name}",
-      "--extra-vars", "ansible_ssh_pass=${var.ssh_password}"//,
-      // "--extra-vars", "repo_username=${var.repo_username}",
-      // "--extra-vars", "repo_password=${var.repo_password}"
+      "--extra-vars", "ansible_ssh_pass=${var.ssh_password}",
+      "--extra-vars", "docker_username=${var.docker_username}",
+      "--extra-vars", "docker_password=${var.docker_password}",
+      "--extra-vars", "repo_username=${var.repo_username}",
+      "--extra-vars", "repo_password=${var.repo_password}"
     ]
   }
 
@@ -36,12 +40,12 @@ build {
     inline = [
       "pwsh -command \"& scripts/Update-OvfConfiguration.ps1 \\",
       " -ApplianceType '${source.name}' \\",
-      " -OVFFile '/scratch/airgapped-k8s/${var.vm_name}-${source.name}.ovf' \"",
+      " -OVFFile '/scratch/bld_${var.vm_name}_${source.name}.ovf' \"",
       "pwsh -file scripts/Update-Manifest.ps1 \\",
-      " -ManifestFileName '/scratch/airgapped-k8s/${var.vm_name}-${source.name}.mf'",
+      " -ManifestFileName '/scratch/bld_${var.vm_name}_${source.name}.mf'",
       "ovftool --acceptAllEulas --allowExtraConfig --overwrite \\",
-      " '/scratch/airgapped-k8s/${var.vm_name}-${source.name}.ovf' \\",
-      " /output/airgapped-k8s.${source.name}.ova"
+      " '/scratch/bld_${var.vm_name}_${source.name}.ovf' \\",
+      " /output/airgapped-k8s-${var.k8s_version}.${source.name}.ova"
     ]
   }
 }

packer/source.pkr.hcl

@@ -56,6 +56,6 @@ source "vsphere-iso" "ubuntu" {
 
   export {
     images                = false
-    output_directory      = "/scratch/airgapped-k8s"
+    output_directory      = "/scratch"
   }
 }

packer/variables.pkr.hcl

@@ -28,3 +28,10 @@ local "iso_authenticatedurl" {
     expression = "https://${var.repo_username}:${var.repo_password}@${var.iso_url}"
     sensitive  = true
 }
+
+variable "docker_username" {}
+variable "docker_password" {
+    sensitive = true
+}
+
+variable "k8s_version" {}

packer/vsphere.auto.pkrvars.hcl

@@ -1,9 +1,9 @@
 vcenter_server         = "bv11-vc.bessems.lan"
 vsphere_username       = "administrator@vsphere.local"
 vsphere_datacenter     = "DeSchakel"
-vsphere_cluster        = "Cluster.Legacy"
-vsphere_host           = "bv11-esx.bessems.lan"
-vsphere_datastore      = "ESX00.SSD01"
+vsphere_cluster        = "Cluster.01"
+vsphere_host           = "bv11-esx02.bessems.lan"
+vsphere_datastore      = "ESX02.SSD02"
 vsphere_folder         = "/Packer"
 vsphere_templatefolder = "/Templates"
 vsphere_network        = "LAN"

scripts/Update-OvfConfiguration.bootstrap.yml

@@ -1,12 +1,16 @@
 DeploymentConfigurations:
 
-- Id: cp1w1
+- Id: cp1w1ws0
   Label: 'Workload-cluster: 1 control-plane node/1 worker node'
   Description: 1 control-plane node/1 worker node
 
-- Id: cp1w2
-  Label: 'Workload-cluster: 1 control-plane node/2 worker nodes'
-  Description: 1 control-plane node/2 worker nodes
+- Id: cp1w1ws1
+  Label: 'Workload-cluster: 1 control-plane node/1 worker node/1 worker-storage node'
+  Description: 1 control-plane node/1 worker node/1 worker-storage node
+
+- Id: core
+  Label: No workload-cluster
+  Description: Only the metacluster is deployed (useful for recovery scenario's)
 
 DynamicDisks:
 
@@ -24,8 +28,9 @@ PropertyCategories:
   - Key: deployment.type
     Type: string
     Value:
-    - cp1w1
-    - cp1w2
+    - cp1w1ws0
+    - cp1w1ws1
+    - core
     UserConfigurable: false
 
 - Name: 1) Meta-cluster
@@ -41,8 +46,8 @@ PropertyCategories:
 
   - key: metacluster.vip
     Type: ip
-    Label: Meta-cluster virtual IP*
-    Description: Meta-cluster control plane endpoint virtual IP
+    Label: Meta-cluster virtual IP address*
+    Description: Meta-cluster control plane endpoint virtual IP address
     DefaultValue: ''
     Configurations: '*'
     UserConfigurable: true
@@ -101,18 +106,18 @@ PropertyCategories:
   - Key: guestinfo.gateway
     Type: ip
     Label: Gateway*
-    Description: ''
+    Description: 'A default route is *required*, use a dummy IP address if there is no actual gateway router present'
     DefaultValue: ''
     Configurations: '*'
     UserConfigurable: true
 
-  - Key: guestinfo.ntpserver
-    Type: string(1..)
-    Label: Time server*
-    Description: A comma-separated list of timeservers
-    DefaultValue: 0.pool.ntp.org,1.pool.ntp.org,2.pool.ntp.org
-    Configurations: '*'
-    UserConfigurable: true
+  # - Key: guestinfo.ntpserver
+  #   Type: string(1..)
+  #   Label: Time server*
+  #   Description: A comma-separated list of timeservers
+  #   DefaultValue: 0.pool.ntp.org,1.pool.ntp.org,2.pool.ntp.org
+  #   Configurations: '*'
+  #   UserConfigurable: true
 
 - Name: 3) Workload-cluster
   ProductProperties:
@@ -122,31 +127,62 @@ PropertyCategories:
     Label: Workload-cluster name*
     Description: ''
     DefaultValue: 'workload-{{ hostname.suffix }}'
-    Configurations: '*'
+    Configurations:
+    - cp1w1ws0
+    - cp1w1ws1
     UserConfigurable: true
 
   - Key: workloadcluster.vip
     Type: ip
-    Label: Workload-cluster virtual IP*
-    Description: Workload-cluster control plane endpoint virtual IP
-    DefaultValue: ''
-    Configurations: '*'
+    Label: Workload-cluster virtual IP address*
+    Description: Workload-cluster control plane endpoint virtual IP address
+    DefaultValue: '0.0.0.0'
+    Configurations:
+    - cp1w1ws0
+    - cp1w1ws1
     UserConfigurable: true
 
   - Key: ippool.startip
     Type: ip
-    Label: Workload-cluster IP-pool start IP*
+    Label: Workload-cluster IP-pool start IP address*
     Description: All nodes for the workload-cluster will be provisioned within this IP pool
-    DefaultValue: ''
-    Configurations: '*'
+    DefaultValue: '0.0.0.0'
+    Configurations:
+    - cp1w1ws0
+    - cp1w1ws1
     UserConfigurable: true
 
   - Key: ippool.endip
     Type: ip
-    Label: Workload-cluster IP-pool end IP*
+    Label: Workload-cluster IP-pool end IP address*
     Description: All nodes for the workload-cluster will be provisioned within this IP pool
-    DefaultValue: ''
-    Configurations: '*'
+    DefaultValue: '0.0.0.0'
+    Configurations:
+    - cp1w1ws0
+    - cp1w1ws1
+    UserConfigurable: true
+
+  - Key: workloadcluster.nodesize
+    Type: string["small", "medium", "large"]
+    Label: Workload-cluster node size*
+    Description: |
+      All worker and worker-storage nodes for the workload-cluster will be provisioned with number of cpu-cores and memory as specified:
+      - SMALL: 2 vCPU/6GB RAM
+      - MEDIUM: 4 vCPU/8GB RAM
+      - LARGE: 8 vCPU/16GB RAM
+    DefaultValue: 'small'
+    Configurations:
+    - cp1w1ws0
+    - cp1w1ws1
+    UserConfigurable: true
+
+  - Key: workloadcluster.additionaldisk
+    Type: int(0..120)
+    Label: Workload-cluster block storage disk size*
+    Description: 'All worker-storage nodes for the workload-cluster will be provisioned with an additional disk of the specified size'
+    DefaultValue: '42'
+    Configurations:
+    - cp1w1ws1
     UserConfigurable: true
 
 - Name: 4) Common
@@ -187,6 +223,33 @@ PropertyCategories:
     Configurations: '*'
     UserConfigurable: true
 
+- Name: 6) Identity provider
+  ProductProperties:
+
+  - Key: ldap.fqdn
+    Type: string(1..)
+    Label: LDAP server FQDN/IP-address*
+    Description: The address of the LDAP server which this bootstrap appliance will perform LDAP queries against.
+    DefaultValue: ''
+    Configurations: '*'
+    UserConfigurable: true
+
+  - Key: ldap.dn
+    Type: string(1..)
+    Label: LDAP bind distinguished name*
+    Description: The distinguished name of the user account used for LDAP queries; for example 'CN=ldapreader,OU=Useraccounts,DC=example,DC=com'
+    DefaultValue: ''
+    Configurations: '*'
+    UserConfigurable: true
+
+  - Key: ldap.password
+    Type: password(1..)
+    Label: LDAP bind password*
+    Description: The password of the user account used for LDAP queries.
+    DefaultValue: ''
+    Configurations: '*'
+    UserConfigurable: true
+
 ---
 Variables:
 - Name: hostname.suffix

scripts/Update-OvfConfiguration.upgrade.yml

@@ -22,8 +22,8 @@ PropertyCategories:
 
   - key: metacluster.vip
     Type: ip
-    Label: Meta-cluster virtual IP*
-    Description: Meta-cluster control plane endpoint virtual IP
+    Label: Meta-cluster virtual IP address*
+    Description: Meta-cluster control plane endpoint virtual IP address
     DefaultValue: ''
     Configurations: '*'
     UserConfigurable: true
@@ -31,7 +31,7 @@ PropertyCategories:
   - Key: metacluster.password
     Type: password(7..)
     Label: Meta-cluster administrator password*
-    Description: 'Needed to authenticate with target meta-cluster'
+    Description: Needed to authenticate with target meta-cluster
     DefaultValue: ''
     Configurations: '*'
     UserConfigurable: true
@@ -87,13 +87,13 @@ PropertyCategories:
     Configurations: '*'
     UserConfigurable: true
 
-  - Key: guestinfo.ntpserver
-    Type: string(1..)
-    Label: Time server*
-    Description: A comma-separated list of timeservers
-    DefaultValue: 0.pool.ntp.org,1.pool.ntp.org,2.pool.ntp.org
-    Configurations: '*'
-    UserConfigurable: true
+  # - Key: guestinfo.ntpserver
+  #   Type: string(1..)
+  #   Label: Time server*
+  #   Description: A comma-separated list of timeservers
+  #   DefaultValue: 0.pool.ntp.org,1.pool.ntp.org,2.pool.ntp.org
+  #   Configurations: '*'
+  #   UserConfigurable: true
 
 - Name: 3) Common
   ProductProperties: