Increase default retries;Add healthcheck

.drone.yml

@@ -13,57 +13,34 @@ volumes:
 steps:
 - name: Debugging information
   image: bv11-cr01.bessems.eu/library/packer-extended
-  pull: always
   commands:
   - ansible --version
   - ovftool --version
   - packer --version
   - yamllint --version
-
+- name: Kubernetes Bootstrap Appliance
-- name: Linting
-  depends_on:
-  - Debugging information
   image: bv11-cr01.bessems.eu/library/packer-extended
   pull: always
   commands:
+  - |
+    sed -i -e "s/<<img-password>>/$${SSH_PASSWORD}/g" \
+      packer/preseed/UbuntuServer22.04/user-data
+  - |
+    export K8S_VERSION=$(yq '.components.clusterapi.workload.version.k8s' < ./ansible/vars/metacluster.yml)
   - |
     yamllint -d "{extends: relaxed, rules: {line-length: disable}}" \
       ansible \
       packer/preseed/UbuntuServer22.04/user-data \
       scripts
-
-- name: Install Ansible Galaxy collections
-  depends_on:
-  - Linting
-  image: bv11-cr01.bessems.eu/library/packer-extended
-  pull: always
-  commands:
   - |
-    ansible-galaxy collection install \
+    ansible-galaxy install \
-      -r ansible/requirements.yml \
+      -r ansible/requirements.yml
-      -p ./ansible/collections
-  volumes:
-  - name: scratch
-    path: /scratch
-
-- name: Kubernetes Bootstrap Appliance
-  depends_on:
-  - Install Ansible Galaxy collections
-  image: bv11-cr01.bessems.eu/library/packer-extended
-  pull: always
-  commands:
-  - |
-    sed -i -e "s/<<img-password>>/$${SSH_PASSWORD}/g" \
-      packer/preseed/UbuntuServer22.04/user-data
-  - |
-    export K8S_VERSION=$(yq '.components.clusterapi.workload.version.k8s' < ./ansible/vars/metacluster.yml)
   - |
     packer init -upgrade \
       ./packer
   - |
     packer validate \
-      -only=vsphere-iso.bootstrap \
+      -var vm_name=$DRONE_BUILD_NUMBER-${DRONE_COMMIT_SHA:0:10} \
-      -var vm_name=${DRONE_BUILD_NUMBER}-${DRONE_COMMIT_SHA:0:10}-$(openssl rand -hex 3) \
       -var docker_username=$${DOCKER_USERNAME} \
       -var docker_password=$${DOCKER_PASSWORD} \
       -var repo_username=$${REPO_USERNAME} \
@@ -75,8 +52,7 @@ steps:
   - |
     packer build \
       -on-error=cleanup -timestamp-ui \
-      -only=vsphere-iso.bootstrap \
+      -var vm_name=$DRONE_BUILD_NUMBER-${DRONE_COMMIT_SHA:0:10} \
-      -var vm_name=${DRONE_BUILD_NUMBER}-${DRONE_COMMIT_SHA:0:10}-$(openssl rand -hex 3) \
       -var docker_username=$${DOCKER_USERNAME} \
       -var docker_password=$${DOCKER_PASSWORD} \
       -var repo_username=$${REPO_USERNAME} \
@@ -104,70 +80,7 @@ steps:
     path: /output
   - name: scratch
     path: /scratch
-
-- name: Kubernetes Upgrade Appliance
-  depends_on:
-  - Install Ansible Galaxy collections
-  image: bv11-cr01.bessems.eu/library/packer-extended
-  pull: alwaysquery(
-  commands:
-  - |
-    sed -i -e "s/<<img-password>>/$${SSH_PASSWORD}/g" \
-      packer/preseed/UbuntuServer22.04/user-data
-  - |
-    export K8S_VERSION=$(yq '.components.clusterapi.workload.version.k8s' < ./ansible/vars/metacluster.yml)
-  - |
-    packer init -upgrade \
-      ./packer
-  - |
-    packer validate \
-      -only=vsphere-iso.upgrade \
-      -var vm_name=${DRONE_BUILD_NUMBER}-${DRONE_COMMIT_SHA:0:10}-$(openssl rand -hex 3) \
-      -var docker_username=$${DOCKER_USERNAME} \
-      -var docker_password=$${DOCKER_PASSWORD} \
-      -var repo_username=$${REPO_USERNAME} \
-      -var repo_password=$${REPO_PASSWORD} \
-      -var ssh_password=$${SSH_PASSWORD} \
-      -var vsphere_password=$${VSPHERE_PASSWORD} \
-      -var k8s_version=$K8S_VERSION \
-      ./packer
-  - |
-    packer build \
-      -on-error=cleanup -timestamp-ui \
-      -only=vsphere-iso.upgrade \
-      -var vm_name=${DRONE_BUILD_NUMBER}-${DRONE_COMMIT_SHA:0:10}-$(openssl rand -hex 3) \
-      -var docker_username=$${DOCKER_USERNAME} \
-      -var docker_password=$${DOCKER_PASSWORD} \
-      -var repo_username=$${REPO_USERNAME} \
-      -var repo_password=$${REPO_PASSWORD} \
-      -var ssh_password=$${SSH_PASSWORD} \
-      -var vsphere_password=$${VSPHERE_PASSWORD} \
-      -var k8s_version=$K8S_VERSION \
-      ./packer
-  environment:
-    DOCKER_USERNAME:
-      from_secret: docker_username
-    DOCKER_PASSWORD:
-      from_secret: docker_password
-    # PACKER_LOG: 1
-    REPO_USERNAME:
-      from_secret: repo_username
-    REPO_PASSWORD:
-      from_secret: repo_password
-    SSH_PASSWORD:
-      from_secret: ssh_password
-    VSPHERE_PASSWORD:
-      from_secret: vsphere_password
-  volumes:
-  - name: output
-    path: /output
-  - name: scratch
-    path: /scratch
-
 - name: Remove temporary resources
-  depends_on:
-  - Kubernetes Bootstrap Appliance
-  - Kubernetes Upgrade Appliance
   image: bv11-cr01.bessems.eu/library/packer-extended
   commands:
   - |

ansible/playbook.yml

@@ -3,7 +3,6 @@
   gather_facts: false
   vars_files:
     - metacluster.yml
-    - workloadcluster.yml
   become: true
   roles:
     - os

ansible/roles/assets/tasks/containerimages.yml

@@ -14,22 +14,13 @@
   loop_control:
     label: "{{ item.dest | basename }}"
 
-- name: Parse metacluster helm charts for container images
+- name: Parse helm charts for container images
   ansible.builtin.shell:
     cmd: "{{ item.value.helm.parse_logic }}"
     chdir: /opt/metacluster/helm-charts/{{ item.key }}
-  register: chartimages_metacluster
+  register: chartimages
   when: item.value.helm is defined
-  loop: "{{ query('ansible.builtin.dict', components) }}"
+  loop: "{{ lookup('ansible.builtin.dict', components) }}"
-  loop_control:
-    label: "{{ item.key }}"
-
-- name: Parse workloadcluster helm charts for container images
-  ansible.builtin.shell:
-    cmd: "{{ item.value.parse_logic }}"
-    chdir: /opt/workloadcluster/helm-charts/{{ item.value.namespace }}/{{ item.key }}
-  register: chartimages_workloadcluster
-  loop: "{{ query('ansible.builtin.dict', downstream.helm_charts) }}"
   loop_control:
     label: "{{ item.key }}"
 
@@ -38,7 +29,7 @@
     containerimages_{{ item.source }}: "{{ item.results }}"
   loop:
     - source: charts
-      results: "{{ (chartimages_metacluster | json_query('results[*].stdout_lines')) + (chartimages_workloadcluster | json_query('results[*].stdout_lines')) | select() | flatten | list }}"
+      results: "{{ chartimages | json_query('results[*].stdout_lines') | select() | flatten | list }}"
     - source: kubeadm
       results: "{{ kubeadmimages.stdout_lines }}"
     - source: manifests

ansible/roles/assets/tasks/git.yml

@@ -0,0 +1,5 @@
+- name: Clone git repository
+  ansible.builtin.git:
+    repo: "{{ platform.gitops.repository.uri }}"
+    version: "{{ platform.gitops.repository.revision }}"
+    dest: /opt/metacluster/git-repositories/gitops

ansible/roles/assets/tasks/helm.yml

@@ -3,29 +3,17 @@
     name: "{{ item.name }}"
     repo_url: "{{ item.url }}"
     state: present
-  loop: "{{ platform.helm_repositories + downstream.helm_repositories }}"
+  loop: "{{ platform.helm_repositories }}"
 
-- name: Fetch helm charts for metacluster
+- name: Fetch helm charts
   ansible.builtin.command:
     cmd: helm fetch {{ item.value.helm.chart }} --untar --version {{ item.value.helm.version }}
     chdir: /opt/metacluster/helm-charts
   when: item.value.helm is defined
-  register: helmcharts_metacluster
+  register: helmcharts
-  loop: "{{ query('ansible.builtin.dict', components) }}"
+  loop: "{{ lookup('ansible.builtin.dict', components) }}"
   loop_control:
     label: "{{ item.key }}"
   retries: 5
   delay: 5
-  until: helmcharts_metacluster is not failed
+  until: helmcharts is not failed
-
-- name: Fetch helm charts for workloadcluster
-  ansible.builtin.command:
-    cmd: helm fetch {{ item.value.chart }} --untardir ./{{ item.value.namespace }} --untar --version {{ item.value.version }}
-    chdir: /opt/workloadcluster/helm-charts
-  register: helmcharts_workloadcluster
-  loop: "{{ query('ansible.builtin.dict', downstream.helm_charts) }}"
-  loop_control:
-    label: "{{ item.key }}"
-  retries: 5
-  delay: 5
-  until: helmcharts_workloadcluster is not failed

ansible/roles/assets/tasks/k3s.yml

@@ -21,7 +21,7 @@
 
 - name: Download K3s install script
   ansible.builtin.get_url:
-    url: https://raw.githubusercontent.com/k3s-io/k3s/{{ platform.k3s.version | urlencode }}/install.sh
+    url: https://get.k3s.io
     dest: /opt/metacluster/k3s/install.sh
     owner: root
     group: root

ansible/roles/assets/tasks/main.yml

@@ -12,13 +12,10 @@
     - /opt/metacluster/cluster-api/infrastructure-vsphere/{{ components.clusterapi.management.version.infrastructure_vsphere }}
     - /opt/metacluster/cluster-api/ipam-in-cluster/{{ components.clusterapi.management.version.ipam_incluster }}
     - /opt/metacluster/container-images
-    - /opt/metacluster/git-repositories
+    - /opt/metacluster/git-repositories/gitops
     - /opt/metacluster/helm-charts
     - /opt/metacluster/k3s
     - /opt/metacluster/kube-vip
-    - /opt/workloadcluster/git-repositories/gitops/charts
-    - /opt/workloadcluster/git-repositories/gitops/values
-    - /opt/workloadcluster/helm-charts
     - /opt/workloadcluster/node-templates
     - /var/lib/rancher/k3s/agent/images
     - /var/lib/rancher/k3s/server/manifests

ansible/roles/assets/tasks/manifests.yml

@@ -2,9 +2,9 @@
 
     - name: Aggregate chart_values into dict
       ansible.builtin.set_fact:
-        metacluster_chartvalues: "{{ metacluster_chartvalues | default({}) | combine({ item.key: { 'chart_values': (item.value.helm.chart_values | from_yaml) } }) }}"
+        chart_values: "{{ chart_values | default({}) | combine({ (item.key | regex_replace('[^A-Za-z0-9]', '')): { 'chart_values': (item.value.helm.chart_values | from_yaml) } }) }}"
       when: item.value.helm.chart_values is defined
-      loop: "{{ query('ansible.builtin.dict', components) }}"
+      loop: "{{ lookup('ansible.builtin.dict', components) }}"
       loop_control:
         label: "{{ item.key }}"
 
@@ -14,29 +14,12 @@
         content: >-
           {{
             { 'components': (
-              metacluster_chartvalues |
+              chart_values |
               combine({ 'clusterapi': components.clusterapi }) |
               combine({ 'kubevip'   : components.kubevip }) )
             } | to_nice_yaml(indent=2, width=4096)
           }}
 
-    - name: Aggregate chart_values into dict
-      ansible.builtin.set_fact:
-        workloadcluster_chartvalues: "{{ workloadcluster_chartvalues | default({}) | combine({ item.key: { 'chart_values': (item.value.chart_values | default('') | from_yaml) } }) }}"
-      # when: item.value.chart_values is defined
-      loop: "{{ query('ansible.builtin.dict', downstream.helm_charts) }}"
-      loop_control:
-        label: "{{ item.key }}"
-
-    - name: Write dict to vars_file
-      ansible.builtin.copy:
-        dest: /opt/firstboot/ansible/vars/workloadcluster.yml
-        content: >-
-          {{
-            { 'downstream_components': ( workloadcluster_chartvalues )
-            } | to_nice_yaml(indent=2, width=4096)
-          }}
-
 - name: Download ClusterAPI manifests
   ansible.builtin.get_url:
     url: "{{ item.url }}"
@@ -82,12 +65,6 @@
   delay: 5
   until: clusterapi_manifests is not failed
 
-- name: Update cluster-template with image tags
-  ansible.builtin.replace:
-    dest: /opt/metacluster/cluster-api/infrastructure-vsphere/{{ components.clusterapi.management.version.infrastructure_vsphere }}/cluster-template.yaml
-    regexp: ':\${CPI_IMAGE_K8S_VERSION}'
-    replace: ":{{ components.clusterapi.management.version.cpi_vsphere }}"
-
 - name: Download kube-vip RBAC manifest
   ansible.builtin.get_url:
     url: https://kube-vip.io/manifests/rbac.yaml
@@ -104,6 +81,6 @@
 #     owner: root
 #     group: root
 #     mode: 0600
-#   loop: "{{ query('ansible.builtin.dict', components) | map(attribute='value.manifests') | list | select('defined') | flatten }}"
+#   loop: "{{ lookup('ansible.builtin.dict', components) | map(attribute='value.manifests') | list | select('defined') | flatten }}"
 #   loop_control:
 #     label: "{{ item.type ~ '/' ~ item.name }}"

ansible/roles/firstboot/files/ansible_payload/bootstrap/playbook.yml

@@ -5,7 +5,6 @@
   vars_files:
     - defaults.yml
     - metacluster.yml
-    - workloadcluster.yml
   # become: true
   roles:
     - vapp

ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/git.yml

@@ -8,7 +8,7 @@
         create_namespace: true
         wait: false
         kubeconfig: "{{ kubeconfig.path }}"
-        values: "{{ components['gitea'].chart_values }}"
+        values: "{{ components.gitea.chart_values }}"
 
     - name: Ensure gitea API availability
       ansible.builtin.uri:
@@ -55,7 +55,6 @@
         force_basic_auth: yes
         body:
           name: token_init_{{ lookup('password', '/dev/null length=5 chars=ascii_letters,digits') }}
-          scopes: ["write:public_key","write:org"]
       register: gitea_api_token
 
     - name: Retrieve existing gitea configuration
@@ -108,30 +107,18 @@
               Authorization: token {{ gitea_api_token.json.sha1 }}
             body: "{{ item.body }}"
           loop:
-            - organization: mc
-              body:
-                name: GitOps.ClusterAPI
-                # auto_init: true
-                # default_branch: main
-                description: ClusterAPI manifests
             - organization: mc
               body:
                 name: GitOps.Config
-                # auto_init: true
+                auto_init: true
-                # default_branch: main
+                default_branch: main
                 description: GitOps manifests
             - organization: wl
               body:
-                name: GitOps.Config
+                name: Template.GitOps.Config
                 # auto_init: true
                 # default_branch: main
                 description: GitOps manifests
-            - organization: wl
-              body:
-                name: GitOps.HelmCharts
-                # auto_init: true
-                # default_branch: main
-                description: Helm charts
           loop_control:
             label: "{{ item.organization ~ '/' ~ item.body.name }}"
 

ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/gitops.yml

@@ -8,7 +8,7 @@
         create_namespace: true
         wait: false
         kubeconfig: "{{ kubeconfig.path }}"
-        values: "{{ components['argo-cd'].chart_values }}"
+        values: "{{ components.argocd.chart_values }}"
 
     - name: Ensure argo-cd API availability
       ansible.builtin.uri:
@@ -39,29 +39,24 @@
         mode: 0600
       vars:
         _template:
-          name: gitrepo-mc-gitopsconfig
+          name: argocd-gitrepo-metacluster
           namespace: argo-cd
-          url: https://git.{{ vapp['metacluster.fqdn'] }}/mc/GitOps.Config.git
+          uid: "{{ lookup('ansible.builtin.password', '/dev/null length=5 chars=ascii_lowercase,digits seed=inventory_hostname') }}"
+          privatekey: "{{ lookup('ansible.builtin.file', '~/.ssh/git_rsa_id') | indent(4, true) }}"
       notify:
         - Apply manifests
 
     - name: Create applicationset
       ansible.builtin.template:
         src: applicationset.j2
-        dest: /var/lib/rancher/k3s/server/manifests/{{ _template.application.name }}-manifest.yaml
+        dest: /var/lib/rancher/k3s/server/manifests/{{ _template.name }}-manifest.yaml
         owner: root
         group: root
         mode: 0600
       vars:
         _template:
-          application:
+          name: argocd-applicationset-metacluster
-            name: applicationset-metacluster
+          namespace: argo-cd
-            namespace: argo-cd
-          cluster:
-            url: https://kubernetes.default.svc
-          repository:
-            url: https://git.{{ vapp['metacluster.fqdn'] }}/mc/GitOps.Config.git
-            revision: main
       notify:
         - Apply manifests
 

ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/init.yml

@@ -78,6 +78,6 @@
     src: registries.j2
   vars:
     _template:
-      registries: "{{ source_registries }}"
+      data: "{{ source_registries }}"
       hv:
         fqdn: "{{ vapp['metacluster.fqdn'] }}"

ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/registry.yml

@@ -8,7 +8,7 @@
         create_namespace: true
         wait: false
         kubeconfig: "{{ kubeconfig.path }}"
-        values: "{{ components['harbor'].chart_values }}"
+        values: "{{ components.harbor.chart_values }}"
 
     - name: Ensure harbor API availability
       ansible.builtin.uri:

ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/storage.yml

@@ -7,7 +7,7 @@
         create_namespace: true
         wait: false
         kubeconfig: "{{ kubeconfig.path }}"
-        values: "{{ components['longhorn'].chart_values }}"
+        values: "{{ components.longhorn.chart_values }}"
 
     - name: Ensure longhorn API availability
       ansible.builtin.uri:

ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/clusterapi.yml

@@ -47,27 +47,41 @@
         resourcepool: "{{ vcenter_info.resourcepool }}"
         folder: "{{ vcenter_info.folder }}"
       cluster:
-        nodetemplate: "{{ nodetemplate_inventorypath }}"
+        nodetemplate: "{{ (components.clusterapi.workload.node_template.url | basename | split('.'))[:-1] | join('.') }}"
         publickey: "{{ vapp['guestinfo.rootsshkey'] }}"
         version: "{{ components.clusterapi.workload.version.k8s }}"
         vip: "{{ vapp['workloadcluster.vip'] }}"
 
-- name: Generate cluster-template kustomization manifest
+- name: WORKAROUND - Update image references to use local registry
+  ansible.builtin.replace:
+    dest: "{{ item }}"
+    regexp: '([ ]+image:[ "]+)(?!({{ _template.pattern }}|"{{ _template.pattern }}))'
+    replace: '\1{{ _template.pattern }}'
+  vars:
+    fileglobs:
+      - "{{ query('ansible.builtin.fileglob', '/opt/metacluster/cluster-api/cni-calico/' ~ components.clusterapi.workload.version.calico ~ '/*.yaml') }}"
+      - "{{ query('ansible.builtin.fileglob', '/opt/metacluster/cluster-api/infrastructure-vsphere/' ~ components.clusterapi.management.version.infrastructure_vsphere ~ '/*.yaml') }}"
+    _template:
+      pattern: registry.{{ vapp['metacluster.fqdn'] }}/library/
+  loop: "{{ fileglobs[0:] | flatten | select }}"
+  loop_control:
+    label: "{{ item | basename }}"
+  when:
+    - item is not search("components.yaml|metadata.yaml")
+
+- name: Generate kustomization template
   ansible.builtin.template:
     src: kustomization.cluster-template.j2
     dest: /opt/metacluster/cluster-api/infrastructure-vsphere/{{ components.clusterapi.management.version.infrastructure_vsphere }}/kustomization.yaml
   vars:
     _template:
+      additionaldisk: "{{ vapp['workloadcluster.additionaldisk'] }}"
       network:
         fqdn: "{{ vapp['metacluster.fqdn'] }}"
         dnsserver: "{{ vapp['guestinfo.dnsserver'] }}"
-      nodesize:
-        cpu: "{{ config.clusterapi.size_matrix[ vapp['workloadcluster.nodesize'] ].cpu }}"
-        memory: "{{ config.clusterapi.size_matrix[ vapp['workloadcluster.nodesize'] ].memory }}"
       rootca: "{{ stepca_cm_certs.resources[0].data['root_ca.crt'] }}"
       runcmds:
         - update-ca-certificates
-      registries: "{{ source_registries }}"
 
 - name: Store custom cluster-template
   ansible.builtin.copy:
@@ -107,8 +121,7 @@
     clustersize: >-
       {{ {
         'controlplane': vapp['deployment.type'] | regex_findall('^cp(\d)+') | first,
-        'worker': vapp['deployment.type'] | regex_findall('w(\d)+') | first,
+        'workers': vapp['deployment.type'] | regex_findall('w(\d)+$') | first
-        'workerstorage': vapp['deployment.type'] | regex_findall('ws(\d)+$') | first
       } }}
 
 - name: Generate workload cluster manifest
@@ -117,51 +130,41 @@
       clusterctl generate cluster \
         {{ vapp['workloadcluster.name'] | lower }} \
         --control-plane-machine-count {{ clustersize.controlplane }} \
-        --worker-machine-count {{ clustersize.worker }} \
+        --worker-machine-count {{ clustersize.workers }} \
         --from ./custom-cluster-template.yaml \
         --config ./clusterctl.yaml \
         --kubeconfig {{ kubeconfig.path }}
     chdir: /opt/metacluster/cluster-api
   register: clusterctl_newcluster
 
-- name: Initialize tempfolder
+- name: Initialize tempfile
   ansible.builtin.tempfile:
-    state: directory
+    state: file
   register: capi_clustermanifest
 
 - name: Save workload cluster manifest
   ansible.builtin.copy:
-    dest: "{{ capi_clustermanifest.path }}/new-cluster.yaml"
+    dest: "{{ capi_clustermanifest.path }}"
     content: "{{ clusterctl_newcluster.stdout }}"
 
 - name: Split manifest into separate files
   ansible.builtin.shell:
     cmd: >-
       kubectl slice \
-        -f {{ capi_clustermanifest.path }}/new-cluster.yaml \
+        -f {{ capi_clustermanifest.path }} \
-        -o {{ capi_clustermanifest.path }}/manifests
+        -o /opt/metacluster/cluster-api/new-cluster
 
-- name: Generate nodepool kustomization manifest
+- name: Cleanup tempfile
-  ansible.builtin.template:
+  ansible.builtin.file:
-    src: kustomization.nodepool.j2
+    path: "{{ capi_clustermanifest.path }}"
-    dest: "{{ capi_clustermanifest.path }}/kustomization.yaml"
+    state: absent
-  vars:
+  when: capi_clustermanifest.path is defined
-    _template:
-      cluster:
-        name: "{{ vapp['workloadcluster.name'] }}"
-      nodepool:
-        size: "{{ clustersize.workerstorage }}"
-        additionaldisk: "{{ vapp['workloadcluster.additionaldisk'] }}"
-
-- name: Store nodepool manifest
-  ansible.builtin.copy:
-    dest: "{{ capi_clustermanifest.path }}/manifests/nodepool-worker-storage.yaml"
-    content: "{{ lookup('kubernetes.core.kustomize', dir=capi_clustermanifest.path) }}"
 
 - name: Create in-cluster IpPool
-  ansible.builtin.template:
+  kubernetes.core.k8s:
-    src: ippool.j2
+    template: ippool.j2
-    dest: "{{ capi_clustermanifest.path }}/manifests/inclusterippool-{{ _template.cluster.name }}.yml"
+    state: present
+    kubeconfig: "{{ kubeconfig.path }}"
   vars:
     _template:
       cluster:
@@ -173,40 +176,6 @@
           prefix: "{{ vapp['guestinfo.prefixlength'] }}"
           gateway: "{{ vapp['guestinfo.gateway'] }}"
 
-- name: Initialize/Push git repository
-  ansible.builtin.shell:
-    cmd: |
-      git init
-      git config --global user.email "administrator@{{ vapp['metacluster.fqdn'] }}"
-      git config --global user.name "administrator"
-      git checkout -b main
-      git add ./manifests
-      git commit -m "Upload manifests"
-      git remote add origin https://git.{{ vapp['metacluster.fqdn'] }}/mc/GitOps.ClusterAPI.git
-      git push https://administrator:{{ vapp['metacluster.password'] | urlencode }}@git.{{ vapp['metacluster.fqdn'] }}/mc/GitOps.ClusterAPI.git --all
-    chdir: "{{ capi_clustermanifest.path }}"
-
-- name: Cleanup tempfolder
-  ansible.builtin.file:
-    path: "{{ capi_clustermanifest.path }}"
-    state: absent
-  when: capi_clustermanifest.path is defined
-
-- name: Configure Cluster API repository
-  ansible.builtin.template:
-    src: gitrepo.j2
-    dest: /var/lib/rancher/k3s/server/manifests/{{ _template.name }}-manifest.yaml
-    owner: root
-    group: root
-    mode: 0600
-  vars:
-    _template:
-      name: gitrepo-mc-gitopsclusterapi
-      namespace: argo-cd
-      url: https://git.{{ vapp['metacluster.fqdn'] }}/mc/GitOps.ClusterAPI.git
-  notify:
-    - Apply manifests
-
 - name: WORKAROUND - Wait for ingress ACME requests to complete
   ansible.builtin.shell:
     cmd: >-
@@ -218,30 +187,13 @@
   retries: "{{ playbook.retries }}"
   delay: "{{ (storage_benchmark | int) * (playbook.delay.medium | int) }}"
 
-- name: Create application
+- name: Apply workload cluster manifest
-  ansible.builtin.template:
+  kubernetes.core.k8s:
-    src: application.j2
+    definition: >-
-    dest: /var/lib/rancher/k3s/server/manifests/{{ _template.application.name }}-manifest.yaml
+      {{ clusterctl_newcluster.stdout }}
-    owner: root
+    wait: true
-    group: root
+    kubeconfig: "{{ kubeconfig.path }}"
-    mode: 0600
+# TODO: move to git repo
-  vars:
-    _template:
-      application:
-        name: application-clusterapi-workloadcluster
-        namespace: argo-cd
-      cluster:
-        name: https://kubernetes.default.svc
-        namespace: default
-      repository:
-        url: https://git.{{ vapp['metacluster.fqdn'] }}/mc/GitOps.ClusterAPI.git
-        path: manifests
-        revision: main
-  notify:
-    - Apply manifests
-
-- name: Trigger handlers
-  ansible.builtin.meta: flush_handlers
 
 - name: Wait for cluster to be available
   ansible.builtin.shell:

ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/gitops.yml

@@ -1,105 +1,37 @@
-- name: Aggregate helm charts from filesystem
+- block:
-  ansible.builtin.find:
-    path: /opt/workloadcluster/helm-charts
-    file_type: directory
-    recurse: false
-  register: helm_charts
 
-- name: Create hard-links to populate new git-repository
+    - name: Generate service account in workload cluster
-  ansible.builtin.shell:
+      kubernetes.core.k8s:
-    cmd: >-
+        template: serviceaccount.j2
-      cp -lr {{ item.path }}/ /opt/workloadcluster/git-repositories/gitops/charts
+        state: present
-  loop: "{{ helm_charts.files }}"
-  loop_control:
-    label: "{{ item.path | basename }}"
 
-- name: Create subfolders
+    - name: Retrieve service account bearer token
-  ansible.builtin.file:
+      kubernetes.core.k8s_info:
-    path: /opt/workloadcluster/git-repositories/gitops/values/{{ item.key }}
+        kind: Secret
-    state: directory
+        name: "{{ _template.account.name }}-secret"
-  loop: "{{ query('ansible.builtin.dict', downstream_components) }}"
+        namespace: "{{ _template.account.namespace }}"
-  loop_control:
+      register: workloadcluster_bearertoken
-    label: "{{ item.key }}"
 
-- name: Write chart values to file
+    - name: Register workload cluster in argo-cd
-  ansible.builtin.copy:
+      kubernetes.core.k8s:
-    dest: /opt/workloadcluster/git-repositories/gitops/values/{{ item.key }}/values.yaml
+        template: cluster.j2
-    content: "{{ item.value.chart_values | default('# Empty') | to_nice_yaml(indent=2, width=4096) }}"
+        state: present
-  loop: "{{ query('ansible.builtin.dict', downstream_components) }}"
+        kubeconfig: "{{ kubeconfig.path }}"
-  loop_control:
+      vars:
-    label: "{{ item.key }}"
+        _template:
+          cluster:
+            name: "{{ vapp['workloadcluster.name'] | lower }}"
+            secret: argocd-cluster-{{ vapp['workloadcluster.name'] | lower }}
+            url: https://{{ vapp['workloadcluster.vip'] }}:6443
+            token: "{{ workloadcluster_bearertoken.resources | json_query('[].data.token') }}"
 
-- name: Initialize/Push git repository
-  ansible.builtin.shell:
-    cmd: |
-      git init
-      git config --global user.email "administrator@{{ vapp['metacluster.fqdn'] }}"
-      git config --global user.name "administrator"
-      git checkout -b main
-      git add .
-      git commit -m "Upload charts"
-      git remote add origin https://git.{{ vapp['metacluster.fqdn'] }}/wl/GitOps.Config.git
-      git push https://administrator:{{ vapp['metacluster.password'] | urlencode }}@git.{{ vapp['metacluster.fqdn'] }}/wl/GitOps.Config.git --all
-    chdir: /opt/workloadcluster/git-repositories/gitops
-
-- name: Retrieve workload-cluster kubeconfig
-  kubernetes.core.k8s_info:
-    kind: Secret
-    name: "{{ vapp['workloadcluster.name'] }}-kubeconfig"
-    namespace: default
-    kubeconfig: "{{ kubeconfig.path }}"
-  register: secret_workloadcluster_kubeconfig
-
-- name: Register workload-cluster in argo-cd
-  kubernetes.core.k8s:
-    template: cluster.j2
-    state: present
-    kubeconfig: "{{ kubeconfig.path }}"
   vars:
     _template:
-      cluster:
+      account:
-        name: "{{ vapp['workloadcluster.name'] | lower }}"
+        name: argocd-sa
-        secret: argocd-cluster-{{ vapp['workloadcluster.name'] | lower }}
+        namespace: default
-        url: https://{{ vapp['workloadcluster.vip'] }}:6443
+      clusterrolebinding:
-      kubeconfig:
+        name: argocd-crb
-        ca: "{{ (secret_workloadcluster_kubeconfig.resources[0].data.value | b64decode | from_yaml).clusters[0].cluster['certificate-authority-data'] }}"
+  module_defaults:
-        certificate: "{{ (secret_workloadcluster_kubeconfig.resources[0].data.value | b64decode | from_yaml).users[0].user['client-certificate-data'] }}"
+    group/k8s:
-        key: "{{ (secret_workloadcluster_kubeconfig.resources[0].data.value | b64decode | from_yaml).users[0].user['client-key-data'] }}"
+      kubeconfig: "{{ capi_kubeconfig.path }}"
-
-- name: Configure workload-cluster GitOps repository
-  ansible.builtin.template:
-    src: gitrepo.j2
-    dest: /var/lib/rancher/k3s/server/manifests/{{ _template.name }}-manifest.yaml
-    owner: root
-    group: root
-    mode: 0600
-  vars:
-    _template:
-      name: gitrepo-wl-gitopsconfig
-      namespace: argo-cd
-      url: https://git.{{ vapp['metacluster.fqdn'] }}/wl/GitOps.Config.git
-  notify:
-    - Apply manifests
-
-- name: Create applicationset
-  ansible.builtin.template:
-    src: applicationset.j2
-    dest: /var/lib/rancher/k3s/server/manifests/{{ _template.application.name }}-manifest.yaml
-    owner: root
-    group: root
-    mode: 0600
-  vars:
-    _template:
-      application:
-        name: applicationset-workloadcluster
-        namespace: argo-cd
-      cluster:
-        url: https://{{ vapp['workloadcluster.vip'] }}:6443
-      repository:
-        url: https://git.{{ vapp['metacluster.fqdn'] }}/wl/GitOps.Config.git
-        revision: main
-  notify:
-    - Apply manifests
-
-- name: Trigger handlers
-  ansible.builtin.meta: flush_handlers

ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/nodetemplates.yml

@@ -1,68 +1,77 @@
 - block:
 
-    - name: Check for existing template on hypervisor
+    - name: Check for existing templates on hypervisor
       community.vmware.vmware_guest_info:
-        name: "{{ (filename | basename | split('.'))[:-1] | join('.') }}"
+        name: "{{ (item | basename | split('.'))[:-1] | join('.') }}"
       register: existing_ova
+      loop: "{{ query('ansible.builtin.fileglob', '/opt/workloadcluster/node-templates/*.ova') | sort }}"
       ignore_errors: yes
 
-    - name: Store inventory path of existing template
+    - name: Parse OVA files for network mappings
-      ansible.builtin.set_fact:
+      ansible.builtin.shell:
-        nodetemplate_inventorypath: "{{ existing_ova.instance.hw_folder ~ '/' ~ existing_ova.instance.hw_name }}"
+        cmd: govc import.spec -json {{ item }}
-      when: existing_ova is not failed
+      environment:
+        GOVC_INSECURE: '1'
+        GOVC_URL: "{{ vapp['hv.fqdn'] }}"
+        GOVC_USERNAME: "{{ vapp['hv.username'] }}"
+        GOVC_PASSWORD: "{{ vapp['hv.password'] }}"
+      register: ova_spec
+      when: existing_ova.results[index] is failed
+      loop: "{{ query('ansible.builtin.fileglob', '/opt/workloadcluster/node-templates/*.ova') | sort }}"
+      loop_control:
+        index_var: index
 
-    - block:
+    - name: Deploy OVA templates on hypervisor
+      community.vmware.vmware_deploy_ovf:
+        cluster: "{{ vcenter_info.cluster }}"
+        datastore: "{{ vcenter_info.datastore }}"
+        name: "{{ (item | basename | split('.'))[:-1] | join('.') }}"
+        networks: "{u'{{ ova_spec.results[index].stdout | from_json | json_query('NetworkMapping[0].Name') }}':u'{{ vcenter_info.network }}'}"
+        allow_duplicates: no
+        power_on: false
+        ovf: "{{ item }}"
+      register: ova_deploy
+      when: existing_ova.results[index] is failed
+      loop: "{{ query('ansible.builtin.fileglob', '/opt/workloadcluster/node-templates/*.ova') | sort }}"
+      loop_control:
+        index_var: index
 
-        - name: Parse OVA file for network mappings
+    - name: Add additional placeholder disk
-          ansible.builtin.shell:
+      community.vmware.vmware_guest_disk:
-            cmd: govc import.spec -json {{ filename }}
+        name: "{{ item.instance.hw_name }}"
-          environment:
+        disk:
-            GOVC_INSECURE: '1'
+          - size: 1Gb
-            GOVC_URL: "{{ vapp['hv.fqdn'] }}"
+            scsi_controller: 1
-            GOVC_USERNAME: "{{ vapp['hv.username'] }}"
+            scsi_type: paravirtual
-            GOVC_PASSWORD: "{{ vapp['hv.password'] }}"
+            unit_number: 0
-          register: ova_spec
+      when: ova_deploy.results[index] is not skipped
+      loop: "{{ ova_deploy.results }}"
+      loop_control:
+        index_var: index
+        label: "{{ item.item }}"
 
-        - name: Deploy OVA template on hypervisor
+    # Disabled to allow disks to be resized; at the cost of cloning speed
-          community.vmware.vmware_deploy_ovf:
+    # - name: Create snapshot on deployed VM's
-            cluster: "{{ vcenter_info.cluster }}"
+    #   community.vmware.vmware_guest_snapshot:
-            datastore: "{{ vcenter_info.datastore }}"
+    #     name: "{{ item.instance.hw_name }}"
-            name: "{{ (filename | basename | split('.'))[:-1] | join('.') }}"
+    #     state: present
-            networks: "{u'{{ ova_spec.stdout | from_json | json_query('NetworkMapping[0].Name') }}':u'{{ vcenter_info.network }}'}"
+    #     snapshot_name: "{{ ansible_date_time.iso8601_basic_short }}-base"
-            allow_duplicates: no
+    #   when: ova_deploy.results[index] is not skipped
-            power_on: false
+    #   loop: "{{ ova_deploy.results }}"
-            ovf: "{{ filename }}"
+    #   loop_control:
-          register: ova_deploy
+    #     index_var: index
+    #     label: "{{ item.item }}"
 
-        - name: Add additional placeholder disk
+    - name: Mark deployed VM's as templates
-          community.vmware.vmware_guest_disk:
+      community.vmware.vmware_guest:
-            name: "{{ ova_deploy.instance.hw_name }}"
+        name: "{{ item.instance.hw_name }}"
-            disk:
+        is_template: yes
-              - size: 1Mb
+      when: ova_deploy.results[index] is not skipped
-                scsi_controller: 1
+      loop: "{{ ova_deploy.results }}"
-                scsi_type: paravirtual
+      loop_control:
-                unit_number: 0
+        index_var: index
+        label: "{{ item.item }}"
 
-        # Disabled to allow disks to be resized; at the cost of cloning speed
-        # - name: Create snapshot on deployed VM
-        #   community.vmware.vmware_guest_snapshot:
-        #     name: "{{ ova_deploy.instance.hw_name }}"
-        #     state: present
-        #     snapshot_name: "{{ ansible_date_time.iso8601_basic_short }}-base"
-
-        - name: Mark deployed VM as templates
-          community.vmware.vmware_guest:
-            name: "{{ ova_deploy.instance.hw_name }}"
-            is_template: yes
-
-        - name: Store inventory path of deployed template
-          ansible.builtin.set_fact:
-            nodetemplate_inventorypath: "{{ ova_deploy.instance.hw_folder ~ '/' ~ ova_deploy.instance.hw_name }}"
-
-      when: existing_ova is failed
-
-  vars:
-    filename: "{{ query('ansible.builtin.fileglob', '/opt/workloadcluster/node-templates/*.ova') | first }}"
   module_defaults:
     group/vmware:
       hostname: "{{ vapp['hv.fqdn'] }}"

ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/application.j2

@@ -1,16 +0,0 @@
-apiVersion: argoproj.io/v1alpha1
-kind: Application
-metadata:
-  name: {{ _template.application.name }}
-  namespace: {{ _template.application.namespace }}
-spec:
-  destination:
-    namespace: {{ _template.cluster.namespace }}
-    server: {{ _template.cluster.name }}
-  project: default
-  source:
-    repoURL: {{ _template.repository.url }}
-    path: {{ _template.repository.path }}
-    targetRevision: {{ _template.repository.revision }}
-  syncPolicy:
-    automated: {}

ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/applicationset.j2

@@ -1,33 +1,28 @@
 apiVersion: argoproj.io/v1alpha1
 kind: ApplicationSet
 metadata:
-  name: {{ _template.application.name }}
+  name: {{ _template.name }}
-  namespace: {{ _template.application.namespace }}
+  namespace: {{ _template.namespace }}
 spec:
   generators:
   - git:
-      repoURL: {{ _template.repository.url }}
+      repoURL: ssh://git@gitea-ssh.gitea.svc.cluster.local/mc/GitOps.Config.git
-      revision: {{ _template.repository.revision }}
+      revision: HEAD
       directories:
-      - path: charts/*/*
+      - path: metacluster-applicationset/*
   template:
     metadata:
-      name: application-{% raw %}{{ path.basename }}{% endraw +%}
+      name: {% raw %}'{{ path.basename }}'{% endraw +%}
     spec:
       project: default
       syncPolicy:
         automated:
           prune: true
           selfHeal: true
-        syncOptions:
+      source:
-        - CreateNamespace=true
+        repoURL: ssh://git@gitea-ssh.gitea.svc.cluster.local/mc/GitOps.Config.git
-      sources:
+        targetRevision: HEAD
-      - repoURL: {{ _template.repository.url }}
-        targetRevision: {{ _template.repository.revision }}
         path: {% raw %}'{{ path }}'{% endraw +%}
-        helm:
-          valueFiles:
-          - /values/{% raw %}{{ path.basename }}{% endraw %}/values.yaml
       destination:
-        server: {{ _template.cluster.url }}
+        server: https://kubernetes.default.svc
-        namespace: {% raw %}'{{ path[1] }}'{% endraw +%}
+        namespace: default

ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/cluster.j2

@@ -11,10 +11,8 @@ stringData:
   server: {{ _template.cluster.url }}
   config: |
     {
+      "bearerToken": "{{ _template.cluster.token }}",
       "tlsClientConfig": {
-        "insecure": false,
+        "insecure": true
-        "caData": "{{ _template.kubeconfig.ca }}",
-        "certData": "{{ _template.kubeconfig.certificate }}",
-        "keyData": "{{ _template.kubeconfig.key }}"
       }
     }

ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/gitrepo.j2

@@ -1,9 +1,13 @@
 apiVersion: v1
 kind: Secret
 metadata:
-  name: {{ _template.name }}
+  name: {{ _template.name }}-{{ _template.uid }}
   namespace: {{ _template.namespace }}
   labels:
     argocd.argoproj.io/secret-type: repository
 stringData:
-  url: {{ _template.url }}
+  url: ssh://git@gitea-ssh.gitea.svc.cluster.local/mc/GitOps.Config.git
+  name: {{ _template.name }}
+  insecure: 'true'
+  sshPrivateKey: |
+{{ _template.privatekey }}

ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/kustomization.cluster-template.j2

@@ -3,36 +3,8 @@ kind: Kustomization
 resources:
 - cluster-template.yaml
 
-patches:
+patchesStrategicMerge:
-- patch: |-
+  - |-
-    apiVersion: v1
-    kind: Secret
-    metadata:
-      name: csi-vsphere-config
-      namespace: '${NAMESPACE}'
-    stringData:
-      data: |
-        apiVersion: v1
-        kind: Secret
-        metadata:
-          name: csi-vsphere-config
-          namespace: kube-system
-        stringData:
-          csi-vsphere.conf: |+
-            [Global]
-            insecure-flag = true
-            thumbprint = "${VSPHERE_TLS_THUMBPRINT}"
-            cluster-id = "${NAMESPACE}/${CLUSTER_NAME}"
-
-            [VirtualCenter "${VSPHERE_SERVER}"]
-            user = "${VSPHERE_USERNAME}"
-            password = "${VSPHERE_PASSWORD}"
-            datacenters = "${VSPHERE_DATACENTER}"
-
-            [Network]
-            public-network = "${VSPHERE_NETWORK}"
-        type: Opaque
-- patch: |-
     apiVersion: controlplane.cluster.x-k8s.io/v1beta1
     kind: KubeadmControlPlane
     metadata:
@@ -42,12 +14,7 @@ patches:
       kubeadmConfigSpec:
         clusterConfiguration:
           imageRepository: registry.{{ _template.network.fqdn }}/kubeadm
-        ntp:
+  - |-
-          enabled: true
-          servers:
-            - 0.nl.pool.ntp.org
-            - 1.nl.pool.ntp.org
-- patch: |-
     apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
     kind: KubeadmConfigTemplate
     metadata:
@@ -58,12 +25,19 @@ patches:
         spec:
           clusterConfiguration:
             imageRepository: registry.{{ _template.network.fqdn }}/kubeadm
-          ntp:
+          diskSetup:
-            enabled: true
+            filesystems:
-            servers:
+            - device: /dev/sdb1
-              - 0.nl.pool.ntp.org
+              filesystem: ext4
-              - 1.nl.pool.ntp.org
+              label: blockstorage
-- patch: |-
+            partitions:
+            - device: /dev/sdb
+              layout: true
+              tableType: gpt
+          mounts:
+          - - LABEL=blockstorage
+            - /mnt/blockstorage
+  - |-
     apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
     kind: KubeadmConfigTemplate
     metadata:
@@ -73,21 +47,6 @@ patches:
       template:
         spec:
           files:
-          - content: |
-                [plugins."io.containerd.grpc.v1.cri".registry]
-                  config_path = "/etc/containerd/certs.d"
-            append: true
-            path: /etc/containerd/config.toml
-{% for registry in _template.registries %}
-          - content: |
-              server = "https://{{ registry }}"
-
-              [host."https://registry.{{ _template.network.fqdn }}/v2/library/{{ registry }}"]
-                capabilities = ["pull", "resolve"]
-                override_path = true
-            owner: root:root
-            path: /etc/containerd/certs.d/{{ registry }}/hosts.toml
-{% endfor %}
           - content: |
               network: {config: disabled}
             owner: root:root
@@ -96,7 +55,7 @@ patches:
               {{ _template.rootca | indent(width=14, first=False) | trim }}
             owner: root:root
             path: /usr/local/share/ca-certificates/root_ca.crt
-- patch: |-
+  - |-
     apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
     kind: VSphereMachineTemplate
     metadata:
@@ -115,7 +74,7 @@ patches:
               nameservers:
               - {{ _template.network.dnsserver }}
               networkName: '${VSPHERE_NETWORK}'
-- patch: |-
+  - |-
     apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
     kind: VSphereMachineTemplate
     metadata:
@@ -124,6 +83,8 @@ patches:
     spec:
       template:
         spec:
+          additionalDisksGiB:
+          - {{ _template.additionaldisk }}
           network:
             devices:
             - dhcp4: false
@@ -135,131 +96,46 @@ patches:
               - {{ _template.network.dnsserver }}
               networkName: '${VSPHERE_NETWORK}'
 
-- target:
+patchesJson6902:
-    group: controlplane.cluster.x-k8s.io
+  - target:
-    version: v1beta1
+      group: controlplane.cluster.x-k8s.io
-    kind: KubeadmControlPlane
+      version: v1beta1
-    name: .*
+      kind: KubeadmControlPlane
-  patch: |-
+      name: .*
-    - op: add
+    patch: |-
-      path: /spec/kubeadmConfigSpec/files/-
+      - op: add
-      value:
+        path: /spec/kubeadmConfigSpec/files/-
-        content: |
+        value:
-            [plugins."io.containerd.grpc.v1.cri".registry]
+          content: |
-              config_path = "/etc/containerd/certs.d"
+            network: {config: disabled}
-        append: true
+          owner: root:root
-        path: /etc/containerd/config.toml
+          path: /etc/cloud/cloud.cfg.d/99-disable-network-config.cfg
-{% for registry in _template.registries %}
+      - op: add
-    - op: add
+        path: /spec/kubeadmConfigSpec/files/-
-      path: /spec/kubeadmConfigSpec/files/-
+        value:
-      value:
+          content: |
-        content: |
+            {{ _template.rootca | indent(width=12, first=False) | trim }}
-          server = "https://{{ registry }}"
+          owner: root:root
-
+          path: /usr/local/share/ca-certificates/root_ca.crt
-          [host."https://registry.{{ _template.network.fqdn }}/v2/library/{{ registry }}"]
+  - target:
-            capabilities = ["pull", "resolve"]
+      group: bootstrap.cluster.x-k8s.io
-            override_path = true
+      version: v1beta1
-        owner: root:root
+      kind: KubeadmConfigTemplate
-        path: /etc/containerd/certs.d/{{ registry }}/hosts.toml
+      name: .*
-{% endfor %}
+    patch: |-
-    - op: add
-      path: /spec/kubeadmConfigSpec/files/-
-      value:
-        content: |
-          network: {config: disabled}
-        owner: root:root
-        path: /etc/cloud/cloud.cfg.d/99-disable-network-config.cfg
-    - op: add
-      path: /spec/kubeadmConfigSpec/files/-
-      value:
-        content: |
-          {{ _template.rootca | indent(width=10, first=False) | trim }}
-        owner: root:root
-        path: /usr/local/share/ca-certificates/root_ca.crt
-- target:
-    group: bootstrap.cluster.x-k8s.io
-    version: v1beta1
-    kind: KubeadmConfigTemplate
-    name: .*
-  patch: |-
 {% for cmd in _template.runcmds %}
-    - op: add
+      - op: add
-      path: /spec/template/spec/preKubeadmCommands/-
+        path: /spec/template/spec/preKubeadmCommands/-
-      value: {{ cmd }}
+        value: {{ cmd }}
 {% endfor %}
-- target:
+  - target:
-    group: controlplane.cluster.x-k8s.io
+      group: controlplane.cluster.x-k8s.io
-    version: v1beta1
+      version: v1beta1
-    kind: KubeadmControlPlane
+      kind: KubeadmControlPlane
-    name: .*
+      name: .*
-  patch: |-
+    patch: |-
 {% for cmd in _template.runcmds %}
-    - op: add
+      - op: add
-      path: /spec/kubeadmConfigSpec/preKubeadmCommands/-
+        path: /spec/kubeadmConfigSpec/preKubeadmCommands/-
-      value: {{ cmd }}
+        value: {{ cmd }}
 {% endfor %}
-
-- target:
-    group: infrastructure.cluster.x-k8s.io
-    version: v1beta1
-    kind: VSphereMachineTemplate
-    name: \${CLUSTER_NAME}
-  patch: |-
-    - op: replace
-      path: /metadata/name
-      value: ${CLUSTER_NAME}-master
-- target:
-    group: controlplane.cluster.x-k8s.io
-    version: v1beta1
-    kind: KubeadmControlPlane
-    name: \${CLUSTER_NAME}
-  patch: |-
-    - op: replace
-      path: /metadata/name
-      value: ${CLUSTER_NAME}-master
-    - op: replace
-      path: /spec/machineTemplate/infrastructureRef/name
-      value: ${CLUSTER_NAME}-master
-- target:
-    group: cluster.x-k8s.io
-    version: v1beta1
-    kind: Cluster
-    name: \${CLUSTER_NAME}
-  patch: |-
-    - op: replace
-      path: /spec/controlPlaneRef/name
-      value: ${CLUSTER_NAME}-master
-
-- target:
-    group: infrastructure.cluster.x-k8s.io
-    version: v1beta1
-    kind: VSphereMachineTemplate
-    name: \${CLUSTER_NAME}-worker
-  patch: |-
-    - op: replace
-      path: /spec/template/spec/numCPUs
-      value: {{ _template.nodesize.cpu }}
-    - op: replace
-      path: /spec/template/spec/memoryMiB
-      value: {{ _template.nodesize.memory }}
-- target:
-    group: cluster.x-k8s.io
-    version: v1beta1
-    kind: MachineDeployment
-    name: \${CLUSTER_NAME}-md-0
-  patch: |-
-    - op: replace
-      path: /metadata/name
-      value: ${CLUSTER_NAME}-worker
-    - op: replace
-      path: /spec/template/spec/bootstrap/configRef/name
-      value: ${CLUSTER_NAME}-worker
-- target:
-    group: bootstrap.cluster.x-k8s.io
-    version: v1beta1
-    kind: KubeadmConfigTemplate
-    name: \${CLUSTER_NAME}-md-0
-  patch: |-
-    - op: replace
-      path: /metadata/name
-      value: ${CLUSTER_NAME}-worker

ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/kustomization.nodepool.j2

@@ -1,83 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-resources:
-- manifests/kubeadmconfigtemplate-{{ _template.cluster.name }}-worker.yaml
-- manifests/machinedeployment-{{ _template.cluster.name }}-worker.yaml
-- manifests/vspheremachinetemplate-{{ _template.cluster.name }}-worker.yaml
-
-patches:
-- patch: |-
-    apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
-    kind: KubeadmConfigTemplate
-    metadata:
-      name: {{ _template.cluster.name }}-worker
-      namespace: default
-    spec:
-      template:
-        spec:
-          diskSetup:
-            filesystems:
-            - device: /dev/sdb1
-              filesystem: ext4
-              label: blockstorage
-            partitions:
-            - device: /dev/sdb
-              layout: true
-              tableType: gpt
-          joinConfiguration:
-            nodeRegistration:
-              kubeletExtraArgs:
-                node-labels: "node.longhorn.io/create-default-disk=true"
-          mounts:
-          - - LABEL=blockstorage
-            - /mnt/blockstorage
-- patch: |-
-    apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
-    kind: VSphereMachineTemplate
-    metadata:
-      name: {{ _template.cluster.name }}-worker
-      namespace: default
-    spec:
-      template:
-        spec:
-          additionalDisksGiB:
-          - {{ _template.nodepool.additionaldisk }}
-
-- target:
-    group: bootstrap.cluster.x-k8s.io
-    version: v1beta1
-    kind: KubeadmConfigTemplate
-    name: {{ _template.cluster.name }}-worker
-  patch: |-
-    - op: replace
-      path: /metadata/name
-      value: {{ _template.cluster.name }}-worker-storage
-
-- target:
-    group: cluster.x-k8s.io
-    version: v1beta1
-    kind: MachineDeployment
-    name: {{ _template.cluster.name }}-worker
-  patch: |-
-    - op: replace
-      path: /metadata/name
-      value: {{ _template.cluster.name }}-worker-storage
-    - op: replace
-      path: /spec/template/spec/bootstrap/configRef/name
-      value: {{ _template.cluster.name }}-worker-storage
-    - op: replace
-      path: /spec/template/spec/infrastructureRef/name
-      value: {{ _template.cluster.name }}-worker-storage
-    - op: replace
-      path: /spec/replicas
-      value: {{ _template.nodepool.size }}
-
-- target:
-    group: infrastructure.cluster.x-k8s.io
-    version: v1beta1
-    kind: VSphereMachineTemplate
-    name: {{ _template.cluster.name }}-worker
-  patch: |-
-    - op: replace
-      path: /metadata/name
-      value: {{ _template.cluster.name }}-worker-storage

ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/serviceaccount.j2

@@ -0,0 +1,27 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ _template.account.name }}
+  namespace: {{ _template.account.namespace }}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ _template.account.name }}-secret
+  namespace: {{ _template.account.namespace }}
+  annotations:
+    kubernetes.io/service-account.name: {{ _template.account.name }}
+type: kubernetes.io/service-account-token
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ _template.clusterrolebinding.name }}
+subjects:
+- kind: ServiceAccount
+  name: {{ _template.account.name }}
+  namespace: {{ _template.account.namespace }}
+roleRef:
+  kind: ClusterRole
+  name: cluster-admin
+  apiGroup: rbac.authorization.k8s.io

ansible/roles/firstboot/files/ansible_payload/common/roles/disks/tasks/main.yml

@@ -28,10 +28,6 @@
   ansible.builtin.set_fact:
     storage_benchmark: "{{ [storage_benchmark, (end_time | int - start_time | int)] | max }}"
 
-- name: Log benchmark actual duration
-  ansible.builtin.debug:
-    msg: "Benchmark actual duration: {{ (end_time | int - start_time | int) }} second(s)"
-
 - name: Mount dynamic disk
   ansible.posix.mount:
     path: /mnt/blockstorage

ansible/roles/firstboot/files/ansible_payload/common/templates/registries.j2

@@ -1,8 +1,8 @@
 mirrors:
-{% for registry in _template.registries %}
+{% for entry in _template.data %}
-  {{ registry }}:
+  {{ entry }}:
     endpoint:
       - https://registry.{{ _template.hv.fqdn }}
     rewrite:
-      "(.*)": "library/{{ registry }}/$1"
+      "(.*)": "library/{{ entry }}/$1"
 {% endfor %}

ansible/roles/firstboot/files/ansible_payload/common/templates/tty.j2

@@ -33,7 +33,7 @@ while /bin/true; do
   echo -e "${PRST}" > /dev/tty1
   echo -e "\n\n\t${DFLT}To manage this appliance, please connect to one of the following:${LCLR}\n" > /dev/tty1
 
-  for c in $( echo "${!COMPONENTS[@]}" | tr ' ' $'\n' | sort); do
+  for c in "${!COMPONENTS[@]}"; do
     STATUS=$(curl -kLs "${COMPONENTS[${c}]}" -o /dev/null -w '%{http_code}')
 
     if [[ "${STATUS}" -eq "200" ]]; then

ansible/roles/firstboot/files/ansible_payload/common/vars/defaults.yml

@@ -8,16 +8,3 @@ playbook:
 
 # This default value is updated during the playbook, based on an I/O intensive operation
 storage_benchmark: 30
-
-config:
-  clusterapi:
-    size_matrix:
-      small:
-        cpu: 2
-        memory: 6144
-      medium:
-        cpu: 4
-        memory: 8192
-      large:
-        cpu: 8
-        memory: 16384

ansible/roles/firstboot/files/ansible_payload/upgrade/roles/decommission/tasks/storage.yml

@@ -22,6 +22,6 @@
       spec:
         numberOfReplicas: {{ (lookup('kubernetes.core.k8s', kind='node', kubeconfig=(kubeconfig.path)) | length | int) - 1 }}
     kubeconfig: "{{ kubeconfig.path }}"
-  loop: "{{ query('kubernetes.core.k8s', api_version='longhorn.io/v1beta2', kind='volume', namespace='longhorn-system', kubeconfig=(kubeconfig.path)) }}"
+  loop: "{{ lookup('kubernetes.core.k8s', api_version='longhorn.io/v1beta2', kind='volume', namespace='longhorn-system', kubeconfig=(kubeconfig.path)) }}"
   loop_control:
     label: "{{ item.metadata.name }}"

ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/git.yml

@@ -7,7 +7,7 @@
         release_namespace: gitea
         wait: false
         kubeconfig: "{{ kubeconfig.path }}"
-        values: "{{ components['gitea'].chart_values }}"
+        values: "{{ components.gitea.chart_values }}"
 
     - name: Ensure gitea API availability
       ansible.builtin.uri:

ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/gitops.yml

@@ -7,7 +7,7 @@
         release_namespace: argo-cd
         wait: false
         kubeconfig: "{{ kubeconfig.path }}"
-        values: "{{ components['argo-cd'].chart_values }}"
+        values: "{{ components.argocd.chart_values }}"
 
     - name: Ensure argo-cd API availability
       ansible.builtin.uri:

ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/registry.yml

@@ -7,7 +7,7 @@
         release_namespace: harbor
         wait: false
         kubeconfig: "{{ kubeconfig.path }}"
-        values: "{{ components['harbor'].chart_values }}"
+        values: "{{ components.harbor.chart_values }}"
 
     - name: Ensure harbor API availability
       ansible.builtin.uri:

ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/storage.yml

@@ -11,7 +11,7 @@
           spec:
             numberOfReplicas: {{ lookup('kubernetes.core.k8s', kind='node', kubeconfig=(kubeconfig.path)) | length | int }}
         kubeconfig: "{{ kubeconfig.path }}"
-      loop: "{{ query('kubernetes.core.k8s', api_version='longhorn.io/v1beta2', kind='volume', namespace='longhorn-system', kubeconfig=(kubeconfig.path)) }}"
+      loop: "{{ lookup('kubernetes.core.k8s', api_version='longhorn.io/v1beta2', kind='volume', namespace='longhorn-system', kubeconfig=(kubeconfig.path)) }}"
       loop_control:
         label: "{{ item.metadata.name }}"
 
@@ -34,7 +34,7 @@
         release_namespace: longhorn-system
         wait: false
         kubeconfig: "{{ kubeconfig.path }}"
-        values: "{{ components['longhorn'].chart_values }}"
+        values: "{{ components.longhorn.chart_values }}"
 
     - name: Ensure longhorn API availability
       ansible.builtin.uri:

ansible/roles/firstboot/files/ansible_payload/upgrade/roles/workloadcluster/tasks/nodetemplates.yml

@@ -1,68 +1,81 @@
 - block:
 
-    - name: Check for existing template on hypervisor
+    - name: Check for existing templates on hypervisor
       community.vmware.vmware_guest_info:
-        name: "{{ (filename | basename | split('.'))[:-1] | join('.') }}"
+        name: "{{ (item | basename | split('.'))[:-1] | join('.') }}"
       register: existing_ova
+      loop: "{{ query('ansible.builtin.fileglob', '/opt/workloadcluster/node-templates/*.ova') | sort }}"
       ignore_errors: yes
 
-    - name: Store inventory path of existing template
+    - name: Parse OVA files for network mappings
-      ansible.builtin.set_fact:
+      ansible.builtin.shell:
-        nodetemplate_inventorypath: "{{ existing_ova.instance.hw_folder ~ '/' ~ existing_ova.instance.hw_name }}"
+        cmd: govc import.spec -json {{ item }}
-      when: existing_ova is not failed
+      environment:
+        GOVC_INSECURE: '1'
+        GOVC_URL: "{{ vapp['hv.fqdn'] }}"
+        GOVC_USERNAME: "{{ vapp['hv.username'] }}"
+        GOVC_PASSWORD: "{{ vapp['hv.password'] }}"
+      register: ova_spec
+      when: existing_ova.results[index] is failed
+      loop: "{{ query('ansible.builtin.fileglob', '/opt/workloadcluster/node-templates/*.ova') | sort }}"
+      loop_control:
+        index_var: index
 
-    - block:
+    - name: Deploy OVA templates on hypervisor
+      community.vmware.vmware_deploy_ovf:
+        cluster: "{{ vcenter_info.cluster }}"
+        datastore: "{{ vcenter_info.datastore }}"
+        folder: "{{ vcenter_info.folder }}"
+        name: "{{ (item | basename | split('.'))[:-1] | join('.') }}"
+        networks: "{u'{{ ova_spec.results[index].stdout | from_json | json_query('NetworkMapping[0].Name') }}':u'{{ vcenter_info.network }}'}"
+        allow_duplicates: no
+        power_on: false
+        ovf: "{{ item }}"
+      register: ova_deploy
+      when: existing_ova.results[index] is failed
+      loop: "{{ query('ansible.builtin.fileglob', '/opt/workloadcluster/node-templates/*.ova') | sort }}"
+      loop_control:
+        index_var: index
 
-        - name: Parse OVA file for network mappings
+    - name: Add vApp properties on deployed VM's
-          ansible.builtin.shell:
+      ansible.builtin.shell:
-            cmd: govc import.spec -json {{ filename }}
+        cmd: >-
-          environment:
+          npp-prepper \
-            GOVC_INSECURE: '1'
+            --server "{{ vapp['hv.fqdn'] }}" \
-            GOVC_URL: "{{ vapp['hv.fqdn'] }}"
+            --username "{{ vapp['hv.username'] }}" \
-            GOVC_USERNAME: "{{ vapp['hv.username'] }}"
+            --password "{{ vapp['hv.password'] }}" \
-            GOVC_PASSWORD: "{{ vapp['hv.password'] }}"
+            vm \
-          register: ova_spec
+            --datacenter "{{ vcenter_info.datacenter }}" \
+            --portgroup "{{ vcenter_info.network }}" \
+            --name "{{ item.instance.hw_name }}"
+      when: existing_ova.results[index] is failed
+      loop: "{{ ova_deploy.results }}"
+      loop_control:
+        index_var: index
+        label: "{{ item.item }}"
 
-        - name: Deploy OVA template on hypervisor
+    - name: Create snapshot on deployed VM's
-          community.vmware.vmware_deploy_ovf:
+      community.vmware.vmware_guest_snapshot:
-            cluster: "{{ vcenter_info.cluster }}"
+        folder: "{{ vcenter_info.folder }}"
-            datastore: "{{ vcenter_info.datastore }}"
+        name: "{{ item.instance.hw_name }}"
-            name: "{{ (filename | basename | split('.'))[:-1] | join('.') }}"
+        state: present
-            networks: "{u'{{ ova_spec.stdout | from_json | json_query('NetworkMapping[0].Name') }}':u'{{ vcenter_info.network }}'}"
+        snapshot_name: "{{ ansible_date_time.iso8601_basic_short }}-base"
-            allow_duplicates: no
+      when: ova_deploy.results[index] is not skipped
-            power_on: false
+      loop: "{{ ova_deploy.results }}"
-            ovf: "{{ filename }}"
+      loop_control:
-          register: ova_deploy
+        index_var: index
+        label: "{{ item.item }}"
 
-        - name: Add additional placeholder disk
+    - name: Mark deployed VM's as templates
-          community.vmware.vmware_guest_disk:
+      community.vmware.vmware_guest:
-            name: "{{ ova_deploy.instance.hw_name }}"
+        name: "{{ item.instance.hw_name }}"
-            disk:
+        is_template: yes
-              - size: 1Gb
+      when: ova_deploy.results[index] is not skipped
-                scsi_controller: 1
+      loop: "{{ ova_deploy.results }}"
-                scsi_type: paravirtual
+      loop_control:
-                unit_number: 0
+        index_var: index
+        label: "{{ item.item }}"
 
-        # Disabled to allow disks to be resized; at the cost of cloning speed
-        # - name: Create snapshot on deployed VM
-        #   community.vmware.vmware_guest_snapshot:
-        #     name: "{{ ova_deploy.instance.hw_name }}"
-        #     state: present
-        #     snapshot_name: "{{ ansible_date_time.iso8601_basic_short }}-base"
-
-        - name: Mark deployed VM as templates
-          community.vmware.vmware_guest:
-            name: "{{ ova_deploy.instance.hw_name }}"
-            is_template: yes
-
-        - name: Store inventory path of deployed template
-          ansible.builtin.set_fact:
-            nodetemplate_inventorypath: "{{ ova_deploy.instance.hw_folder ~ '/' ~ ova_deploy.instance.hw_name }}"
-
-      when: existing_ova is failed
-
-  vars:
-    filename: "{{ query('ansible.builtin.fileglob', '/opt/metacluster/node-templates/*.ova') | first }}"
   module_defaults:
     group/vmware:
       hostname: "{{ vapp['hv.fqdn'] }}"
@@ -70,4 +83,3 @@
       username: "{{ vapp['hv.username'] }}"
       password: "{{ vapp['hv.password'] }}"
       datacenter: "{{ vcenter_info.datacenter }}"
-      folder: "{{ vcenter_info.folder }}"

ansible/roles/firstboot/files/ansible_payload/upgrade/roles/workloadcluster/tasks/registry.yml

@@ -1,5 +1,9 @@
 - block:
 
+  - name: Lookup kubeadm container images
+    ansible.builtin.set_fact:
+      kubeadm_images: "{{ lookup('ansible.builtin.file', '/opt/metacluster/cluster-api/imagelist').splitlines() }}"
+
   - name: Copy kubeadm container images to dedicated project
     ansible.builtin.uri:
       url: https://registry.{{ vapp['metacluster.fqdn'] }}/api/v2.0/projects/kubeadm/repositories/{{ ( item | regex_findall('([^:/]+)') )[-2] }}/artifacts?from=library/{{ item | replace('/', '%2F') | replace(':', '%3A') }}
@@ -8,7 +12,7 @@
         Authorization: "Basic {{ ('admin:' ~ vapp['metacluster.password']) | b64encode }}"
       body:
         from: "{{ item }}"
-    loop: "{{ lookup('ansible.builtin.file', '/opt/metacluster/cluster-api/imagelist').splitlines() }}"
+    loop: "{{ kubeadm_images }}"
 
   module_defaults:
     ansible.builtin.uri:

ansible/vars/metacluster.yml

@@ -1,7 +1,13 @@
 platform:
 
   k3s:
-    version: v1.26.5+k3s1
+    version: v1.25.7+k3s1
+
+  gitops:
+    repository:
+      uri: https://code.spamasaurus.com/djpbessems/GitOps.MetaCluster.git
+      # revision: v0.1.0
+      revision: HEAD
 
   packaged_components:
     - name: traefik
@@ -33,12 +39,8 @@ platform:
   helm_repositories:
     - name: argo
       url: https://argoproj.github.io/argo-helm
-    - name: authentik
+    - name: dex
-      url: https://charts.goauthentik.io
+      url: https://charts.dexidp.io
-    # - name: codecentric
-    #   url: https://codecentric.github.io/helm-charts
-    # - name: dex
-    #   url: https://charts.dexidp.io
     - name: gitea-charts
       url: https://dl.gitea.io/charts/
     - name: harbor
@@ -56,7 +58,7 @@ components:
 
   argo-cd:
     helm:
-      version: 5.34.6  # (= ArgoCD v2.7.3)
+      version: 5.24.0  # (= ArgoCD v2.6.3)
       chart: argo/argo-cd
       parse_logic: helm template . | yq --no-doc eval '.. | .image? | select(.)' | sort -u | awk '!/ /'
       chart_values: !unsafe |
@@ -71,35 +73,9 @@ components:
             hosts:
               - gitops.{{ vapp['metacluster.fqdn'] }}
 
-  authentik:
-    helm:
-      version: 2023.3.1
-      chart: authentik/authentik
-      parse_logic: helm template . --set postgresql.enabled=true,redis.enabled=true | yq --no-doc eval '.. | .image? | select(.)' | sort -u | awk '!/ /'
-      chart_values: !unsafe |
-        authentik:
-          avatars: none
-          secret_key: "{{ lookup('ansible.builtin.password', '/dev/null length=64 chars=ascii_lowercase,digits seed=' ~ vapp['guestinfo.hostname']) }}"
-          postgresql:
-            password: "{{ lookup('ansible.builtin.password', '/dev/null length=32 chars=ascii_lowercase,digits seed=' ~ vapp['guestinfo.hostname']) }}"
-        env:
-          AUTHENTIK_BOOTSTRAP_PASSWORD: "{{ vapp['metacluster.password'] }}"
-        ingress:
-          enabled: true
-          hosts:
-            - host: auth.{{ vapp['metacluster.fqdn'] }}
-              paths:
-                - path: "/"
-                  pathType: Prefix
-        postgresql:
-          enabled: true
-          postgresqlPassword: "{{ lookup('ansible.builtin.password', '/dev/null length=32 chars=ascii_lowercase,digits seed=' ~ vapp['guestinfo.hostname']) }}"
-        redis:
-          enabled: true
-
   cert-manager:
     helm:
-      version: 1.12.1
+      version: 1.11.0
       chart: jetstack/cert-manager
       parse_logic: helm template . | yq --no-doc eval '.. | .image? | select(.)' | sort -u | awk '!/ /'
       # chart_values: !unsafe |
@@ -109,71 +85,69 @@ components:
     management:
       version:
         # Must match the version referenced at `dependencies.static_binaries[.filename==clusterctl].url`
-        base: v1.4.1
+        base: v1.3.5
         # Must match the version referenced at `components.cert-manager.helm.version`
-        cert_manager: v1.11.1
+        cert_manager: v1.11.0
-        infrastructure_vsphere: v1.6.1
+        infrastructure_vsphere: v1.5.3
         ipam_incluster: v0.1.0-alpha.2
-        # Refer to `https://console.cloud.google.com/gcr/images/cloud-provider-vsphere/GLOBAL/cpi/release/manager` for available tags
-        cpi_vsphere: v1.26.2
     workload:
       version:
-        calico: v3.26.0
+        calico: v3.25.0
-        k8s: v1.26.5
+        k8s: v1.25.8
       node_template:
-        url: https://{{ repo_username }}:{{ repo_password }}@sn.itch.fyi/Repository/rel/ubuntu-2204-kube-v1.26.5.ova
+        url: https://{{ repo_username }}:{{ repo_password }}@sn.itch.fyi/Repository/rel/ubuntu-2004-kube-v1.25.8.ova
 
-  # dex:
+  dex:
-  #   helm:
+    helm:
-  #     version: 0.13.0 # (= Dex 2.35.3)
+      version: 0.13.0 # (= Dex 2.35.3)
-  #     chart: dex/dex
+      chart: dex/dex
-  #     parse_logic: helm template . | yq --no-doc eval '.. | .image? | select(.)' | sort -u | awk '!/ /'
+      parse_logic: helm template . | yq --no-doc eval '.. | .image? | select(.)' | sort -u | awk '!/ /'
-  #     chart_values: !unsafe |
+      chart_values: !unsafe |
-  #       config:
+        config:
-  #         connectors:
+          connectors:
-  #           - type: ldap
+            - type: ldap
-  #             id: ldap
+              id: ldap
-  #             name: "LDAP"
+              name: "LDAP"
-  #             config:
+              config:
-  #               host: "{{ vapp['ldap.fqdn'] }}:636"
+                host: "{{ vapp['ldap.fqdn'] }}:636"
-  #               insecureNoSSL: false
+                insecureNoSSL: false
-  #               insecureSkipVerify: true
+                insecureSkipVerify: true
-  #               bindDN: "{{ vapp['ldap.dn'] }}"
+                bindDN: "{{ vapp['ldap.dn'] }}"
-  #               bindPW: "{{ vapp['ldap.password'] }}"
+                bindPW: "{{ vapp['ldap.password'] }}"
 
-  #               usernamePrompt: "Username"
+                usernamePrompt: "Username"
-  #               userSearch:
+                userSearch:
-  #                 baseDN: OU=Administrators,OU=Useraccounts,DC=bessems,DC=eu
+                  baseDN: OU=Administrators,OU=Useraccounts,DC=bessems,DC=eu
-  #                 filter: "(objectClass=person)"
+                  filter: "(objectClass=person)"
-  #                 username: userPrincipalName
+                  username: userPrincipalName
-  #                 idAttr: DN
+                  idAttr: DN
-  #                 emailAttr: userPrincipalName
+                  emailAttr: userPrincipalName
-  #                 nameAttr: cn
+                  nameAttr: cn
 
-  #               groupSearch:
+                groupSearch:
-  #                 baseDN: OU=Roles,OU=Groups,DC=bessems,DC=eu
+                  baseDN: OU=Roles,OU=Groups,DC=bessems,DC=eu
-  #                 filter: "(objectClass=group)"
+                  filter: "(objectClass=group)"
-  #                 userMatchers:
+                  userMatchers:
-  #                 - userAttr: DN
+                  - userAttr: DN
-  #                   groupAttr: member
+                    groupAttr: member
-  #                 nameAttr: cn
+                  nameAttr: cn
-  #         enablePasswordDB: true
+          enablePasswordDB: true
-  #         issuer: https://oidc.{{ vapp['metacluster.fqdn'] }}
+          issuer: https://oidc.{{ vapp['metacluster.fqdn'] }}
-  #         storage:
+          storage:
-  #           type: kubernetes
+            type: kubernetes
-  #           config:
+            config:
-  #             inCluster: true
+              inCluster: true
-  #       ingress:
+        ingress:
-  #         enabled: true
+          enabled: true
-  #         hosts:
+          hosts:
-  #           - host: oidc.{{ vapp['metacluster.fqdn'] }}
+            - host: oidc.{{ vapp['metacluster.fqdn'] }}
-  #             paths:
+              paths:
-  #               - path: /
+                - path: /
-  #                 pathType: Prefix
+                  pathType: Prefix
 
   gitea:
     helm:
-      version: v8.3.0 # (= Gitea v1.19.3)
+      version: v7.0.2 # (= Gitea v1.18.3)
       chart: gitea-charts/gitea
       parse_logic: helm template . | yq --no-doc eval '.. | .image? | select(.)' | sort -u | sed '/:/!s/$/:latest/'
       chart_values: !unsafe |
@@ -204,7 +178,7 @@ components:
 
   harbor:
     helm:
-      version: 1.12.1  # (= Harbor v2.8.1)
+      version: 1.11.0  # (= Harbor v2.7.0)
       chart: harbor/harbor
       parse_logic: helm template . | yq --no-doc eval '.. | .image? | select(.)' | sort -u | awk '!/ /'
       chart_values: !unsafe |
@@ -225,41 +199,9 @@ components:
             registry:
               size: 25Gi
 
-  # keycloakx:
-  #   helm:
-  #     version: 2.1.1  # (= Keycloak 20.0.3)
-  #     chart: codecentric/keycloakx
-  #     parse_logic: helm template . | yq --no-doc eval '.. | .image? | select(.)' | sort -u | awk '!/ /'
-  #     chart_values: !unsafe |
-  #       command:
-  #         - "/opt/keycloak/bin/kc.sh"
-  #         - "start"
-  #         - "--http-enabled=true"
-  #         - "--http-port=8080"
-  #         - "--hostname-strict=false"
-  #         - "--hostname-strict-https=false"
-  #       extraEnv: |
-  #         - name: KEYCLOAK_ADMIN
-  #           value: admin
-  #         - name: KEYCLOAK_ADMIN_PASSWORD
-  #           value: {{ vapp['metacluster.password'] }}
-  #         - name: KC_PROXY
-  #           value: "passthrough"
-  #         - name: JAVA_OPTS_APPEND
-  #           value: >-
-  #             -Djgroups.dns.query={% raw %}{{ include "keycloak.fullname" . }}{% endraw %}-headless
-  #       ingress:
-  #         enabled: true
-  #         rules:
-  #           - host: keycloak.{{ vapp['metacluster.fqdn'] }}
-  #             paths:
-  #               - path: /
-  #                 pathType: Prefix
-  #         tls: []
-
   kube-prometheus-stack:
     helm:
-      version: 46.5.0  # (= Prometheus version v0.65.1)
+      version: 45.2.0
       chart: prometheus-community/kube-prometheus-stack
       parse_logic: helm template . | yq --no-doc eval '.. | .image? | select(.)' | sort -u | awk '!/ /'
       chart_values: !unsafe |
@@ -270,11 +212,11 @@ components:
 
   kubevip:
     # Must match the version referenced at `dependencies.container_images`
-    version: v0.6.0
+    version: v0.5.8
 
   longhorn:
     helm:
-      version: 1.4.2
+      version: 1.4.0
       chart: longhorn/longhorn
       parse_logic: cat values.yaml | yq eval '.. | select(has("repository")) | .repository + ":" + .tag'
       chart_values: !unsafe |
@@ -290,7 +232,7 @@ components:
 
   step-certificates:
     helm:
-      version: 1.23.2+5  # (= step-ca v0.23.2)
+      version: 1.23.0
       chart: smallstep/step-certificates
       parse_logic: helm template . | yq --no-doc eval '.. | .image? | select(.)' | sed '/:/!s/$/:latest/' | sort -u
       chart_values: !unsafe |
@@ -320,7 +262,7 @@ dependencies:
   container_images:
     # This should match the image tag referenced at `platform.packaged_components[.name==traefik].config`
     - busybox:1
-    - ghcr.io/kube-vip/kube-vip:v0.6.0
+    - ghcr.io/kube-vip/kube-vip:v0.5.8
     # The following list is generated by running the following commands:
     #   $ clusterctl init -i vsphere:<version> [...]
     #   $ clusterctl generate cluster <name> [...] | yq eval '.data.data' | yq --no-doc eval '.. | .image? | select(.)' | sort -u
@@ -334,25 +276,25 @@ dependencies:
 
   static_binaries:
     - filename: clusterctl
-      url: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.4.1/clusterctl-linux-amd64
+      url: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.3.5/clusterctl-linux-amd64
     - filename: govc
-      url: https://github.com/vmware/govmomi/releases/download/v0.30.4/govc_Linux_x86_64.tar.gz
+      url: https://github.com/vmware/govmomi/releases/download/v0.29.0/govc_Linux_x86_64.tar.gz
       archive: compressed
     - filename: helm
-      url: https://get.helm.sh/helm-v3.12.0-linux-amd64.tar.gz
+      url: https://get.helm.sh/helm-v3.10.2-linux-amd64.tar.gz
       archive: compressed
       extra_opts: --strip-components=1
     - filename: kubectl-slice
-      url: https://github.com/patrickdappollonio/kubectl-slice/releases/download/v1.2.6/kubectl-slice_linux_x86_64.tar.gz
+      url: https://github.com/patrickdappollonio/kubectl-slice/releases/download/v1.2.5/kubectl-slice_linux_x86_64.tar.gz
       archive: compressed
     - filename: skopeo
-      url: https://code.spamasaurus.com/api/packages/djpbessems/generic/skopeo/v1.12.0/skopeo_linux_amd64
+      url: https://code.spamasaurus.com/api/packages/djpbessems/generic/skopeo/v1.11.1/skopeo_linux_amd64
     - filename: step
-      url: https://dl.step.sm/gh-release/cli/gh-release-header/v0.23.2/step_linux_0.23.2_amd64.tar.gz
+      url: https://dl.step.sm/gh-release/cli/gh-release-header/v0.23.0/step_linux_0.23.0_amd64.tar.gz
       archive: compressed
       extra_opts: --strip-components=2
     - filename: yq
-      url: http://github.com/mikefarah/yq/releases/download/v4.34.1/yq_linux_amd64
+      url: http://github.com/mikefarah/yq/releases/download/v4.30.5/yq_linux_amd64
 
   packages:
     apt:

ansible/vars/workloadcluster.yml

@@ -1,27 +0,0 @@
-downstream:
-
-  helm_repositories:
-    - name: longhorn
-      url: https://charts.longhorn.io
-    - name: sealed-secrets
-      url: https://bitnami-labs.github.io/sealed-secrets
-
-  helm_charts:
-
-    longhorn:
-      version: 1.4.2
-      chart: longhorn/longhorn
-      namespace: longhorn-system
-      parse_logic: cat values.yaml | yq eval '.. | select(has("repository")) | .repository + ":" + .tag'
-      chart_values: !unsafe |
-        defaultSettings:
-          createDefaultDiskLabeledNodes: true
-          defaultDataPath: /mnt/blockstorage
-
-    sealed-secrets:
-      version: 2.9.0  # (= Sealed Secrets v0.21.0)
-      chart: sealed-secrets/sealed-secrets
-      namespace: sealed-secrets
-      parse_logic: helm template . | yq --no-doc eval '.. | .image? | select(.)' | sort -u | awk '!/ /'
-      # chart_values: !unsafe |
-      #   # Empty

packer/build.pkr.hcl

@@ -6,12 +6,22 @@ packer {
 build {
   source "vsphere-iso.ubuntu" {
     name = "bootstrap"
-    vm_name = "bld_${var.vm_name}_bootstrap"
+    vm_name = "ova.bootstrap-${var.vm_name}"
+
+    export {
+      images                = false
+      output_directory      = "/scratch/airgapped-k8s/bootstrap"
+    }
   }
 
   source "vsphere-iso.ubuntu" {
     name = "upgrade"
-    vm_name = "bld_${var.vm_name}_upgrade"
+    vm_name = "ova.upgrade-${var.vm_name}"
+
+    export {
+      images                = false
+      output_directory      = "/scratch/airgapped-k8s/upgrade"
+    }
   }
 
   provisioner "ansible" {
@@ -24,8 +34,6 @@ build {
       "PYTHONUNBUFFERED=1"
     ]
     use_proxy        = "false"
-    collections_path = "ansible/collections"
-
     extra_arguments  = [
       "--extra-vars", "appliancetype=${source.name}",
       "--extra-vars", "ansible_ssh_pass=${var.ssh_password}",
@@ -40,11 +48,11 @@ build {
     inline = [
       "pwsh -command \"& scripts/Update-OvfConfiguration.ps1 \\",
       " -ApplianceType '${source.name}' \\",
-      " -OVFFile '/scratch/bld_${var.vm_name}_${source.name}.ovf' \"",
+      " -OVFFile '/scratch/airgapped-k8s/${source.name}/ova.${source.name}-${var.vm_name}.ovf' \"",
       "pwsh -file scripts/Update-Manifest.ps1 \\",
-      " -ManifestFileName '/scratch/bld_${var.vm_name}_${source.name}.mf'",
+      " -ManifestFileName '/scratch/airgapped-k8s/${source.name}/ova.${source.name}-${var.vm_name}.mf'",
       "ovftool --acceptAllEulas --allowExtraConfig --overwrite \\",
-      " '/scratch/bld_${var.vm_name}_${source.name}.ovf' \\",
+      " '/scratch/airgapped-k8s/${source.name}/ova.${source.name}-${var.vm_name}.ovf' \\",
       " /output/airgapped-k8s-${var.k8s_version}.${source.name}.ova"
     ]
   }

packer/source.pkr.hcl

@@ -53,9 +53,4 @@ source "vsphere-iso" "ubuntu" {
   shutdown_timeout        = "5m"
 
   remove_cdrom            = true
-
-  export {
-    images                = false
-    output_directory      = "/scratch"
-  }
 }

scripts/Update-OvfConfiguration.bootstrap.yml

@@ -1,12 +1,12 @@
 DeploymentConfigurations:
 
-- Id: cp1w1ws0
+- Id: cp1w1
   Label: 'Workload-cluster: 1 control-plane node/1 worker node'
   Description: 1 control-plane node/1 worker node
 
-- Id: cp1w1ws1
+- Id: cp1w2
-  Label: 'Workload-cluster: 1 control-plane node/1 worker node/1 worker-storage node'
+  Label: 'Workload-cluster: 1 control-plane node/2 worker nodes'
-  Description: 1 control-plane node/1 worker node/1 worker-storage node
+  Description: 1 control-plane node/2 worker nodes
 
 - Id: core
   Label: No workload-cluster
@@ -28,8 +28,8 @@ PropertyCategories:
   - Key: deployment.type
     Type: string
     Value:
-    - cp1w1ws0
+    - cp1w1
-    - cp1w1ws1
+    - cp1w2
     - core
     UserConfigurable: false
 
@@ -128,61 +128,61 @@ PropertyCategories:
     Description: ''
     DefaultValue: 'workload-{{ hostname.suffix }}'
     Configurations:
-    - cp1w1ws0
+    - cp1w1
-    - cp1w1ws1
+    - cp1w2
     UserConfigurable: true
 
   - Key: workloadcluster.vip
     Type: ip
     Label: Workload-cluster virtual IP address*
     Description: Workload-cluster control plane endpoint virtual IP address
-    DefaultValue: '0.0.0.0'
+    DefaultValue: ''
     Configurations:
-    - cp1w1ws0
+    - cp1w1
-    - cp1w1ws1
+    - cp1w2
     UserConfigurable: true
 
   - Key: ippool.startip
     Type: ip
     Label: Workload-cluster IP-pool start IP address*
     Description: All nodes for the workload-cluster will be provisioned within this IP pool
-    DefaultValue: '0.0.0.0'
+    DefaultValue: ''
     Configurations:
-    - cp1w1ws0
+    - cp1w1
-    - cp1w1ws1
+    - cp1w2
     UserConfigurable: true
 
   - Key: ippool.endip
     Type: ip
     Label: Workload-cluster IP-pool end IP address*
     Description: All nodes for the workload-cluster will be provisioned within this IP pool
-    DefaultValue: '0.0.0.0'
+    DefaultValue: ''
     Configurations:
-    - cp1w1ws0
+    - cp1w1
-    - cp1w1ws1
+    - cp1w2
     UserConfigurable: true
 
   - Key: workloadcluster.nodesize
-    Type: string["small", "medium", "large"]
+    Type: string["small", "medium"]
     Label: Workload-cluster node size*
     Description: |
-      All worker and worker-storage nodes for the workload-cluster will be provisioned with number of cpu-cores and memory as specified:
+      All worker-nodes for the workload-cluster will be provisioned with number of cpu-cores and memory as specified:
       - SMALL: 2 vCPU/6GB RAM
       - MEDIUM: 4 vCPU/8GB RAM
-      - LARGE: 8 vCPU/16GB RAM
     DefaultValue: 'small'
     Configurations:
-    - cp1w1ws0
+    - cp1w1
-    - cp1w1ws1
+    - cp1w2
     UserConfigurable: true
 
   - Key: workloadcluster.additionaldisk
     Type: int(0..120)
     Label: Workload-cluster block storage disk size*
-    Description: 'All worker-storage nodes for the workload-cluster will be provisioned with an additional disk of the specified size'
+    Description: 'All worker-nodes for the workload-cluster will be provisioned with an additional disk of the specified size'
-    DefaultValue: '42'
+    DefaultValue: '20'
     Configurations:
-    - cp1w1ws1
+    - cp1w1
+    - cp1w2
     UserConfigurable: true
 
 - Name: 4) Common