From ec6f712427784694208b2e624a0e9dc50d27e11e Mon Sep 17 00:00:00 2001
From: Danny Bessems <danny@bessems.eu>
Date: Thu, 23 Mar 2023 16:55:11 +0100
Subject: [PATCH 01/51] Add healthcheck;Improve console healthchecks;Increase
 default retries

---
 .../workloadcluster/tasks/clusterapi.yml      |  4 +++-
 .../common/roles/tty/tasks/main.yml           | 19 +++++++++++++++++++
 .../ansible_payload/common/templates/tty.j2   | 15 +++++++++------
 .../ansible_payload/common/vars/defaults.yml  |  2 +-
 4 files changed, 32 insertions(+), 8 deletions(-)

diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/clusterapi.yml b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/clusterapi.yml
index 29154cb..4281c54 100644
--- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/clusterapi.yml
+++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/clusterapi.yml
@@ -99,7 +99,7 @@
         --kubeconfig {{ kubeconfig.path }}
     chdir: /opt/metacluster/cluster-api
 
-- name: Ensure CAPI/CAPV controller availability
+- name: Ensure controller availability
   kubernetes.core.k8s_info:
     kind: Deployment
     name: "{{ item.name }}"
@@ -107,6 +107,8 @@
     wait: true
     kubeconfig: "{{ kubeconfig.path }}"
   loop:
+    - name: caip-in-cluster-controller-manager
+      namespace: caip-in-cluster-system
     - name: capi-controller-manager
       namespace: capi-system
     - name: capv-controller-manager
diff --git a/ansible/roles/firstboot/files/ansible_payload/common/roles/tty/tasks/main.yml b/ansible/roles/firstboot/files/ansible_payload/common/roles/tty/tasks/main.yml
index 81cd533..9bf2ec9 100644
--- a/ansible/roles/firstboot/files/ansible_payload/common/roles/tty/tasks/main.yml
+++ b/ansible/roles/firstboot/files/ansible_payload/common/roles/tty/tasks/main.yml
@@ -19,6 +19,25 @@
         executable: /opt/firstboot/tty.sh
         workingdir: /tmp/
       metacluster:
+        components:
+          - name: ArgoCD
+            url: https://gitops.${FQDN}
+            healthcheck: https://gitops.${FQDN}
+          - name: Gitea
+            url: https://git.${FQDN}
+            healthcheck: https://git.${FQDN}
+          - name: Harbor
+            url: https://registry.${FQDN}
+            healthcheck: https://registry.${FQDN}
+          - name: Longhorn
+            url: https://storage.${FQDN}
+            healthcheck: https://storage.${FQDN}
+          - name: StepCA
+            url: ''
+            healthcheck: https://ca.${FQDN}/health
+          - name: Traefik
+            url: https://ingress.${FQDN}
+            healthcheck: https://ingress.${FQDN}
         fqdn: "{{ vapp['metacluster.fqdn'] }}"
         vip: "{{ vapp['metacluster.vip'] }}"
   loop:
diff --git a/ansible/roles/firstboot/files/ansible_payload/common/templates/tty.j2 b/ansible/roles/firstboot/files/ansible_payload/common/templates/tty.j2
index d9f1c49..8eb0e83 100644
--- a/ansible/roles/firstboot/files/ansible_payload/common/templates/tty.j2
+++ b/ansible/roles/firstboot/files/ansible_payload/common/templates/tty.j2
@@ -12,11 +12,14 @@ DFLT='\033[0m'        # Reset colour
 LCLR='\033[K'         # Clear to end of line
 PRST='\033[0;0H'      # Reset cursor position
 
-# COMPONENTS=('ca' 'ingress' 'storage' 'registry' 'git' 'gitops')
-COMPONENTS=('storage' 'registry' 'git' 'gitops')
 FQDN='{{ _template.metacluster.fqdn }}'
 IPADDRESS='{{ _template.metacluster.vip }}'
 
+declare -A COMPONENTS
+{% for component in _template.metacluster.components %}
+COMPONENTS["{{ component.name }}\t({{ component.url }})"]="{{ component.healthcheck }}"
+{% endfor %}
+
 I=0
 
 while /bin/true; do
@@ -30,13 +33,13 @@ while /bin/true; do
   echo -e "${PRST}" > /dev/tty1
   echo -e "\n\n\t${DFLT}To manage this appliance, please connect to one of the following:${LCLR}\n" > /dev/tty1
 
-  for c in "${COMPONENTS[@]}"; do
-    STATUS=$(curl -ks "https://${c}.${FQDN}" -o /dev/null -w '%{http_code}')
+  for c in "${!COMPONENTS[@]}"; do
+    STATUS=$(curl -kLs "${COMPONENTS[${c}]}" -o /dev/null -w '%{http_code}')
 
     if [[ "${STATUS}" -eq "200" ]]; then
-      echo -e "\t [${BGRN}+${DFLT}] ${BBLU}https://${c}.${FQDN}${DFLT}${LCLR}" > /dev/tty1
+      echo -e "\t [${BGRN}+${DFLT}] ${BBLU}${c}${DFLT}${LCLR}" > /dev/tty1
     else
-      echo -e "\t [${BRED}-${DFLT}] ${BBLU}https://${c}.${FQDN}${DFLT}${LCLR}" > /dev/tty1
+      echo -e "\t [${BRED}-${DFLT}] ${BBLU}${c}${DFLT}${LCLR}" > /dev/tty1
     fi
   done
 
diff --git a/ansible/roles/firstboot/files/ansible_payload/common/vars/defaults.yml b/ansible/roles/firstboot/files/ansible_payload/common/vars/defaults.yml
index 8cf3e3c..c3b1df9 100644
--- a/ansible/roles/firstboot/files/ansible_payload/common/vars/defaults.yml
+++ b/ansible/roles/firstboot/files/ansible_payload/common/vars/defaults.yml
@@ -1,5 +1,5 @@
 playbook:
-  retries: 5
+  retries: 10
   delay:
     # These values are multiplied with the value of `storage_benchmark`
     long: 2

From fa0b72a9038338ca65c1c352f4db248a8f92d0f8 Mon Sep 17 00:00:00 2001
From: Danny Bessems <danny@bessems.eu>
Date: Sat, 25 Mar 2023 16:58:23 +0100
Subject: [PATCH 02/51] Remove git repo logic; Debug ova templates

---
 ansible/playbook.yml                              |  1 +
 ansible/roles/assets/tasks/git.yml                |  5 -----
 ansible/roles/assets/tasks/main.yml               |  4 ++--
 ansible/roles/assets/tasks/nodetemplates.yml      |  2 +-
 .../roles/workloadcluster/tasks/nodetemplates.yml | 11 ++++++++---
 .../common/roles/disks/tasks/main.yml             |  4 ++++
 .../roles/workloadcluster/tasks/nodetemplates.yml |  6 +++---
 ansible/vars/metacluster.yml                      | 12 ++++++------
 ansible/vars/workloadcluster.yml                  | 15 +++++++++++++++
 9 files changed, 40 insertions(+), 20 deletions(-)
 delete mode 100644 ansible/roles/assets/tasks/git.yml
 create mode 100644 ansible/vars/workloadcluster.yml

diff --git a/ansible/playbook.yml b/ansible/playbook.yml
index d0763e8..4939995 100644
--- a/ansible/playbook.yml
+++ b/ansible/playbook.yml
@@ -3,6 +3,7 @@
   gather_facts: false
   vars_files:
     - metacluster.yml
+    - workloadcluster.yml
   become: true
   roles:
     - os
diff --git a/ansible/roles/assets/tasks/git.yml b/ansible/roles/assets/tasks/git.yml
deleted file mode 100644
index f4d2f8f..0000000
--- a/ansible/roles/assets/tasks/git.yml
+++ /dev/null
@@ -1,5 +0,0 @@
-- name: Clone git repository
-  ansible.builtin.git:
-    repo: "{{ platform.gitops.repository.uri }}"
-    version: "{{ platform.gitops.repository.revision }}"
-    dest: /opt/metacluster/git-repositories/gitops
diff --git a/ansible/roles/assets/tasks/main.yml b/ansible/roles/assets/tasks/main.yml
index d565b5c..13d7529 100644
--- a/ansible/roles/assets/tasks/main.yml
+++ b/ansible/roles/assets/tasks/main.yml
@@ -12,11 +12,11 @@
     - /opt/metacluster/cluster-api/infrastructure-vsphere/{{ components.clusterapi.management.version.infrastructure_vsphere }}
     - /opt/metacluster/cluster-api/ipam-in-cluster/{{ components.clusterapi.management.version.ipam_incluster }}
     - /opt/metacluster/container-images
-    - /opt/metacluster/git-repositories/gitops
+    - /opt/metacluster/git-repositories
     - /opt/metacluster/helm-charts
     - /opt/metacluster/k3s
     - /opt/metacluster/kube-vip
-    - /opt/workloadcluster/node-templates
+    - /opt/metacluster/node-templates
     - /var/lib/rancher/k3s/agent/images
     - /var/lib/rancher/k3s/server/manifests
 
diff --git a/ansible/roles/assets/tasks/nodetemplates.yml b/ansible/roles/assets/tasks/nodetemplates.yml
index eb7b93c..52104f2 100644
--- a/ansible/roles/assets/tasks/nodetemplates.yml
+++ b/ansible/roles/assets/tasks/nodetemplates.yml
@@ -1,4 +1,4 @@
 - name: Download node-template image
   ansible.builtin.uri:
     url: "{{ components.clusterapi.workload.node_template.url }}"
-    dest: /opt/workloadcluster/node-templates/{{ components.clusterapi.workload.node_template.url | basename}}
+    dest: /opt/metacluster/node-templates/{{ components.clusterapi.workload.node_template.url | basename}}
diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/nodetemplates.yml b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/nodetemplates.yml
index fa0545c..4674ca2 100644
--- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/nodetemplates.yml
+++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/nodetemplates.yml
@@ -4,7 +4,7 @@
       community.vmware.vmware_guest_info:
         name: "{{ (item | basename | split('.'))[:-1] | join('.') }}"
       register: existing_ova
-      loop: "{{ query('ansible.builtin.fileglob', '/opt/workloadcluster/node-templates/*.ova') | sort }}"
+      loop: "{{ query('ansible.builtin.fileglob', '/opt/metacluster/node-templates/*.ova') | first }}"
       ignore_errors: yes
 
     - name: Parse OVA files for network mappings
@@ -17,7 +17,7 @@
         GOVC_PASSWORD: "{{ vapp['hv.password'] }}"
       register: ova_spec
       when: existing_ova.results[index] is failed
-      loop: "{{ query('ansible.builtin.fileglob', '/opt/workloadcluster/node-templates/*.ova') | sort }}"
+      loop: "{{ query('ansible.builtin.fileglob', '/opt/metacluster/node-templates/*.ova') | first }}"
       loop_control:
         index_var: index
 
@@ -32,10 +32,15 @@
         ovf: "{{ item }}"
       register: ova_deploy
       when: existing_ova.results[index] is failed
-      loop: "{{ query('ansible.builtin.fileglob', '/opt/workloadcluster/node-templates/*.ova') | sort }}"
+      loop: "{{ query('ansible.builtin.fileglob', '/opt/metacluster/node-templates/*.ova') | first }}"
       loop_control:
         index_var: index
 
+    - debug:
+        msg: "{{ existing_ova.results | first }}"
+    - debug:
+        msg: "{{ ova_deploy.results | first }}"
+
     - name: Add additional placeholder disk
       community.vmware.vmware_guest_disk:
         name: "{{ item.instance.hw_name }}"
diff --git a/ansible/roles/firstboot/files/ansible_payload/common/roles/disks/tasks/main.yml b/ansible/roles/firstboot/files/ansible_payload/common/roles/disks/tasks/main.yml
index c74bcec..62b80ef 100644
--- a/ansible/roles/firstboot/files/ansible_payload/common/roles/disks/tasks/main.yml
+++ b/ansible/roles/firstboot/files/ansible_payload/common/roles/disks/tasks/main.yml
@@ -28,6 +28,10 @@
   ansible.builtin.set_fact:
     storage_benchmark: "{{ [storage_benchmark, (end_time | int - start_time | int)] | max }}"
 
+- name: Log benchmark actual duration
+  ansible.builtin.debug:
+    msg: "Benchmark actual duration: {{ (end_time | int - start_time | int) }} second(s)"
+
 - name: Mount dynamic disk
   ansible.posix.mount:
     path: /mnt/blockstorage
diff --git a/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/workloadcluster/tasks/nodetemplates.yml b/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/workloadcluster/tasks/nodetemplates.yml
index 2060474..38ab4ea 100644
--- a/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/workloadcluster/tasks/nodetemplates.yml
+++ b/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/workloadcluster/tasks/nodetemplates.yml
@@ -4,7 +4,7 @@
       community.vmware.vmware_guest_info:
         name: "{{ (item | basename | split('.'))[:-1] | join('.') }}"
       register: existing_ova
-      loop: "{{ query('ansible.builtin.fileglob', '/opt/workloadcluster/node-templates/*.ova') | sort }}"
+      loop: "{{ query('ansible.builtin.fileglob', '/opt/workloadcluster/node-templates/*.ova') | first }}"
       ignore_errors: yes
 
     - name: Parse OVA files for network mappings
@@ -17,7 +17,7 @@
         GOVC_PASSWORD: "{{ vapp['hv.password'] }}"
       register: ova_spec
       when: existing_ova.results[index] is failed
-      loop: "{{ query('ansible.builtin.fileglob', '/opt/workloadcluster/node-templates/*.ova') | sort }}"
+      loop: "{{ query('ansible.builtin.fileglob', '/opt/workloadcluster/node-templates/*.ova') | first }}"
       loop_control:
         index_var: index
 
@@ -33,7 +33,7 @@
         ovf: "{{ item }}"
       register: ova_deploy
       when: existing_ova.results[index] is failed
-      loop: "{{ query('ansible.builtin.fileglob', '/opt/workloadcluster/node-templates/*.ova') | sort }}"
+      loop: "{{ query('ansible.builtin.fileglob', '/opt/workloadcluster/node-templates/*.ova') | first }}"
       loop_control:
         index_var: index
 
diff --git a/ansible/vars/metacluster.yml b/ansible/vars/metacluster.yml
index b5b1e90..e9b7a0f 100644
--- a/ansible/vars/metacluster.yml
+++ b/ansible/vars/metacluster.yml
@@ -3,11 +3,11 @@ platform:
   k3s:
     version: v1.26.2+k3s1
 
-  gitops:
-    repository:
-      uri: https://code.spamasaurus.com/djpbessems/GitOps.MetaCluster.git
-      # revision: v0.1.0
-      revision: HEAD
+  # gitops:
+  #   repository:
+  #     uri: https://code.spamasaurus.com/djpbessems/GitOps.MetaCluster.git
+  #     # revision: v0.1.0
+  #     revision: HEAD
 
   packaged_components:
     - name: traefik
@@ -216,7 +216,7 @@ components:
 
   longhorn:
     helm:
-      version: 1.4.0
+      version: 1.4.1
       chart: longhorn/longhorn
       parse_logic: cat values.yaml | yq eval '.. | select(has("repository")) | .repository + ":" + .tag'
       chart_values: !unsafe |
diff --git a/ansible/vars/workloadcluster.yml b/ansible/vars/workloadcluster.yml
new file mode 100644
index 0000000..ae82752
--- /dev/null
+++ b/ansible/vars/workloadcluster.yml
@@ -0,0 +1,15 @@
+downstream:
+
+  helm_repositories:
+    - name: longhorn
+      url: https://charts.longhorn.io
+
+  helm_charts:
+
+    longhorn:
+      version: 1.4.1
+      chart: longhorn/longhorn
+      parse_logic: cat values.yaml | yq eval '.. | select(has("repository")) | .repository + ":" + .tag'
+      chart_values: !unsafe |
+        defaultSettings:
+          defaultDataPath: /mnt/blockstorage

From 89fd23f66ad3f766237454a0f24e13fc30f7446f Mon Sep 17 00:00:00 2001
From: Danny Bessems <danny@bessems.eu>
Date: Sat, 25 Mar 2023 19:13:11 +0100
Subject: [PATCH 03/51] Reference node template by inventory path

---
 .../workloadcluster/tasks/clusterapi.yml      |   2 +-
 .../workloadcluster/tasks/nodetemplates.yml   | 120 ++++++++---------
 .../workloadcluster/tasks/nodetemplates.yml   | 122 ++++++++----------
 3 files changed, 109 insertions(+), 135 deletions(-)

diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/clusterapi.yml b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/clusterapi.yml
index 4281c54..a6bea8f 100644
--- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/clusterapi.yml
+++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/clusterapi.yml
@@ -47,7 +47,7 @@
         resourcepool: "{{ vcenter_info.resourcepool }}"
         folder: "{{ vcenter_info.folder }}"
       cluster:
-        nodetemplate: "{{ (components.clusterapi.workload.node_template.url | basename | split('.'))[:-1] | join('.') }}"
+        nodetemplate: "{{ nodetemplate_inventorypath }}"
         publickey: "{{ vapp['guestinfo.rootsshkey'] }}"
         version: "{{ components.clusterapi.workload.version.k8s }}"
         vip: "{{ vapp['workloadcluster.vip'] }}"
diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/nodetemplates.yml b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/nodetemplates.yml
index 4674ca2..e867b67 100644
--- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/nodetemplates.yml
+++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/nodetemplates.yml
@@ -1,82 +1,68 @@
 - block:
 
-    - name: Check for existing templates on hypervisor
+    - name: Check for existing template on hypervisor
       community.vmware.vmware_guest_info:
-        name: "{{ (item | basename | split('.'))[:-1] | join('.') }}"
+        name: "{{ (filename | basename | split('.'))[:-1] | join('.') }}"
       register: existing_ova
-      loop: "{{ query('ansible.builtin.fileglob', '/opt/metacluster/node-templates/*.ova') | first }}"
       ignore_errors: yes
 
-    - name: Parse OVA files for network mappings
-      ansible.builtin.shell:
-        cmd: govc import.spec -json {{ item }}
-      environment:
-        GOVC_INSECURE: '1'
-        GOVC_URL: "{{ vapp['hv.fqdn'] }}"
-        GOVC_USERNAME: "{{ vapp['hv.username'] }}"
-        GOVC_PASSWORD: "{{ vapp['hv.password'] }}"
-      register: ova_spec
-      when: existing_ova.results[index] is failed
-      loop: "{{ query('ansible.builtin.fileglob', '/opt/metacluster/node-templates/*.ova') | first }}"
-      loop_control:
-        index_var: index
+    - name: Store inventory path of existing template
+      ansible.builtin.set_fact:
+        nodetemplate_inventorypath: "{{ existing_ova.instance.hw_folder ~ '/' ~ existing_ova.instance.hw_name }}"
+      when: existing_ova is not failed
 
-    - name: Deploy OVA templates on hypervisor
-      community.vmware.vmware_deploy_ovf:
-        cluster: "{{ vcenter_info.cluster }}"
-        datastore: "{{ vcenter_info.datastore }}"
-        name: "{{ (item | basename | split('.'))[:-1] | join('.') }}"
-        networks: "{u'{{ ova_spec.results[index].stdout | from_json | json_query('NetworkMapping[0].Name') }}':u'{{ vcenter_info.network }}'}"
-        allow_duplicates: no
-        power_on: false
-        ovf: "{{ item }}"
-      register: ova_deploy
-      when: existing_ova.results[index] is failed
-      loop: "{{ query('ansible.builtin.fileglob', '/opt/metacluster/node-templates/*.ova') | first }}"
-      loop_control:
-        index_var: index
+    - block:
 
-    - debug:
-        msg: "{{ existing_ova.results | first }}"
-    - debug:
-        msg: "{{ ova_deploy.results | first }}"
+        - name: Parse OVA file for network mappings
+          ansible.builtin.shell:
+            cmd: govc import.spec -json {{ filename }}
+          environment:
+            GOVC_INSECURE: '1'
+            GOVC_URL: "{{ vapp['hv.fqdn'] }}"
+            GOVC_USERNAME: "{{ vapp['hv.username'] }}"
+            GOVC_PASSWORD: "{{ vapp['hv.password'] }}"
+          register: ova_spec
 
-    - name: Add additional placeholder disk
-      community.vmware.vmware_guest_disk:
-        name: "{{ item.instance.hw_name }}"
-        disk:
-          - size: 1Gb
-            scsi_controller: 1
-            scsi_type: paravirtual
-            unit_number: 0
-      when: ova_deploy.results[index] is not skipped
-      loop: "{{ ova_deploy.results }}"
-      loop_control:
-        index_var: index
-        label: "{{ item.item }}"
+        - name: Deploy OVA template on hypervisor
+          community.vmware.vmware_deploy_ovf:
+            cluster: "{{ vcenter_info.cluster }}"
+            datastore: "{{ vcenter_info.datastore }}"
+            name: "{{ (filename | basename | split('.'))[:-1] | join('.') }}"
+            networks: "{u'{{ ova_spec.stdout | from_json | json_query('NetworkMapping[0].Name') }}':u'{{ vcenter_info.network }}'}"
+            allow_duplicates: no
+            power_on: false
+            ovf: "{{ filename }}"
+          register: ova_deploy
 
-    # Disabled to allow disks to be resized; at the cost of cloning speed
-    # - name: Create snapshot on deployed VM's
-    #   community.vmware.vmware_guest_snapshot:
-    #     name: "{{ item.instance.hw_name }}"
-    #     state: present
-    #     snapshot_name: "{{ ansible_date_time.iso8601_basic_short }}-base"
-    #   when: ova_deploy.results[index] is not skipped
-    #   loop: "{{ ova_deploy.results }}"
-    #   loop_control:
-    #     index_var: index
-    #     label: "{{ item.item }}"
+        - name: Add additional placeholder disk
+          community.vmware.vmware_guest_disk:
+            name: "{{ ova_deploy.instance.hw_name }}"
+            disk:
+              - size: 1Gb
+                scsi_controller: 1
+                scsi_type: paravirtual
+                unit_number: 0
 
-    - name: Mark deployed VM's as templates
-      community.vmware.vmware_guest:
-        name: "{{ item.instance.hw_name }}"
-        is_template: yes
-      when: ova_deploy.results[index] is not skipped
-      loop: "{{ ova_deploy.results }}"
-      loop_control:
-        index_var: index
-        label: "{{ item.item }}"
+        # Disabled to allow disks to be resized; at the cost of cloning speed
+        # - name: Create snapshot on deployed VM
+        #   community.vmware.vmware_guest_snapshot:
+        #     name: "{{ ova_deploy.instance.hw_name }}"
+        #     state: present
+        #     snapshot_name: "{{ ansible_date_time.iso8601_basic_short }}-base"
 
+        - name: Mark deployed VM as templates
+          community.vmware.vmware_guest:
+            name: "{{ ova_deploy.instance.hw_name }}"
+            is_template: yes
+
+        - name: Store inventory path of deployed template
+          ansible.builtin.set_fact:
+            nodetemplate_inventorypath: "{{ ova_deploy.instance.hw_folder ~ '/' ~ ova_deploy.instance.hw_name }}"
+
+      when: existing_ova is failed
+
+  vars:
+    filename: "{{ query('ansible.builtin.fileglob', '/opt/metacluster/node-templates/*.ova') | first }}"
   module_defaults:
     group/vmware:
       hostname: "{{ vapp['hv.fqdn'] }}"
diff --git a/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/workloadcluster/tasks/nodetemplates.yml b/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/workloadcluster/tasks/nodetemplates.yml
index 38ab4ea..e867b67 100644
--- a/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/workloadcluster/tasks/nodetemplates.yml
+++ b/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/workloadcluster/tasks/nodetemplates.yml
@@ -1,81 +1,68 @@
 - block:
 
-    - name: Check for existing templates on hypervisor
+    - name: Check for existing template on hypervisor
       community.vmware.vmware_guest_info:
-        name: "{{ (item | basename | split('.'))[:-1] | join('.') }}"
+        name: "{{ (filename | basename | split('.'))[:-1] | join('.') }}"
       register: existing_ova
-      loop: "{{ query('ansible.builtin.fileglob', '/opt/workloadcluster/node-templates/*.ova') | first }}"
       ignore_errors: yes
 
-    - name: Parse OVA files for network mappings
-      ansible.builtin.shell:
-        cmd: govc import.spec -json {{ item }}
-      environment:
-        GOVC_INSECURE: '1'
-        GOVC_URL: "{{ vapp['hv.fqdn'] }}"
-        GOVC_USERNAME: "{{ vapp['hv.username'] }}"
-        GOVC_PASSWORD: "{{ vapp['hv.password'] }}"
-      register: ova_spec
-      when: existing_ova.results[index] is failed
-      loop: "{{ query('ansible.builtin.fileglob', '/opt/workloadcluster/node-templates/*.ova') | first }}"
-      loop_control:
-        index_var: index
+    - name: Store inventory path of existing template
+      ansible.builtin.set_fact:
+        nodetemplate_inventorypath: "{{ existing_ova.instance.hw_folder ~ '/' ~ existing_ova.instance.hw_name }}"
+      when: existing_ova is not failed
 
-    - name: Deploy OVA templates on hypervisor
-      community.vmware.vmware_deploy_ovf:
-        cluster: "{{ vcenter_info.cluster }}"
-        datastore: "{{ vcenter_info.datastore }}"
-        folder: "{{ vcenter_info.folder }}"
-        name: "{{ (item | basename | split('.'))[:-1] | join('.') }}"
-        networks: "{u'{{ ova_spec.results[index].stdout | from_json | json_query('NetworkMapping[0].Name') }}':u'{{ vcenter_info.network }}'}"
-        allow_duplicates: no
-        power_on: false
-        ovf: "{{ item }}"
-      register: ova_deploy
-      when: existing_ova.results[index] is failed
-      loop: "{{ query('ansible.builtin.fileglob', '/opt/workloadcluster/node-templates/*.ova') | first }}"
-      loop_control:
-        index_var: index
+    - block:
 
-    - name: Add vApp properties on deployed VM's
-      ansible.builtin.shell:
-        cmd: >-
-          npp-prepper \
-            --server "{{ vapp['hv.fqdn'] }}" \
-            --username "{{ vapp['hv.username'] }}" \
-            --password "{{ vapp['hv.password'] }}" \
-            vm \
-            --datacenter "{{ vcenter_info.datacenter }}" \
-            --portgroup "{{ vcenter_info.network }}" \
-            --name "{{ item.instance.hw_name }}"
-      when: existing_ova.results[index] is failed
-      loop: "{{ ova_deploy.results }}"
-      loop_control:
-        index_var: index
-        label: "{{ item.item }}"
+        - name: Parse OVA file for network mappings
+          ansible.builtin.shell:
+            cmd: govc import.spec -json {{ filename }}
+          environment:
+            GOVC_INSECURE: '1'
+            GOVC_URL: "{{ vapp['hv.fqdn'] }}"
+            GOVC_USERNAME: "{{ vapp['hv.username'] }}"
+            GOVC_PASSWORD: "{{ vapp['hv.password'] }}"
+          register: ova_spec
 
-    - name: Create snapshot on deployed VM's
-      community.vmware.vmware_guest_snapshot:
-        folder: "{{ vcenter_info.folder }}"
-        name: "{{ item.instance.hw_name }}"
-        state: present
-        snapshot_name: "{{ ansible_date_time.iso8601_basic_short }}-base"
-      when: ova_deploy.results[index] is not skipped
-      loop: "{{ ova_deploy.results }}"
-      loop_control:
-        index_var: index
-        label: "{{ item.item }}"
+        - name: Deploy OVA template on hypervisor
+          community.vmware.vmware_deploy_ovf:
+            cluster: "{{ vcenter_info.cluster }}"
+            datastore: "{{ vcenter_info.datastore }}"
+            name: "{{ (filename | basename | split('.'))[:-1] | join('.') }}"
+            networks: "{u'{{ ova_spec.stdout | from_json | json_query('NetworkMapping[0].Name') }}':u'{{ vcenter_info.network }}'}"
+            allow_duplicates: no
+            power_on: false
+            ovf: "{{ filename }}"
+          register: ova_deploy
 
-    - name: Mark deployed VM's as templates
-      community.vmware.vmware_guest:
-        name: "{{ item.instance.hw_name }}"
-        is_template: yes
-      when: ova_deploy.results[index] is not skipped
-      loop: "{{ ova_deploy.results }}"
-      loop_control:
-        index_var: index
-        label: "{{ item.item }}"
+        - name: Add additional placeholder disk
+          community.vmware.vmware_guest_disk:
+            name: "{{ ova_deploy.instance.hw_name }}"
+            disk:
+              - size: 1Gb
+                scsi_controller: 1
+                scsi_type: paravirtual
+                unit_number: 0
 
+        # Disabled to allow disks to be resized; at the cost of cloning speed
+        # - name: Create snapshot on deployed VM
+        #   community.vmware.vmware_guest_snapshot:
+        #     name: "{{ ova_deploy.instance.hw_name }}"
+        #     state: present
+        #     snapshot_name: "{{ ansible_date_time.iso8601_basic_short }}-base"
+
+        - name: Mark deployed VM as templates
+          community.vmware.vmware_guest:
+            name: "{{ ova_deploy.instance.hw_name }}"
+            is_template: yes
+
+        - name: Store inventory path of deployed template
+          ansible.builtin.set_fact:
+            nodetemplate_inventorypath: "{{ ova_deploy.instance.hw_folder ~ '/' ~ ova_deploy.instance.hw_name }}"
+
+      when: existing_ova is failed
+
+  vars:
+    filename: "{{ query('ansible.builtin.fileglob', '/opt/metacluster/node-templates/*.ova') | first }}"
   module_defaults:
     group/vmware:
       hostname: "{{ vapp['hv.fqdn'] }}"
@@ -83,3 +70,4 @@
       username: "{{ vapp['hv.username'] }}"
       password: "{{ vapp['hv.password'] }}"
       datacenter: "{{ vcenter_info.datacenter }}"
+      folder: "{{ vcenter_info.folder }}"

From 5113dd5b6cddd0e2e2152f506b1b26bbc4a87d1a Mon Sep 17 00:00:00 2001
From: Danny Bessems <danny@bessems.eu>
Date: Sat, 25 Mar 2023 23:01:23 +0100
Subject: [PATCH 04/51] Set default values to optional vapp properties

---
 scripts/Update-OvfConfiguration.bootstrap.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scripts/Update-OvfConfiguration.bootstrap.yml b/scripts/Update-OvfConfiguration.bootstrap.yml
index caa0dac..3204984 100644
--- a/scripts/Update-OvfConfiguration.bootstrap.yml
+++ b/scripts/Update-OvfConfiguration.bootstrap.yml
@@ -136,7 +136,7 @@ PropertyCategories:
     Type: ip
     Label: Workload-cluster virtual IP address*
     Description: Workload-cluster control plane endpoint virtual IP address
-    DefaultValue: ''
+    DefaultValue: '0.0.0.0'
     Configurations:
     - cp1w1
     - cp1w2
@@ -146,7 +146,7 @@ PropertyCategories:
     Type: ip
     Label: Workload-cluster IP-pool start IP address*
     Description: All nodes for the workload-cluster will be provisioned within this IP pool
-    DefaultValue: ''
+    DefaultValue: '0.0.0.0'
     Configurations:
     - cp1w1
     - cp1w2
@@ -156,7 +156,7 @@ PropertyCategories:
     Type: ip
     Label: Workload-cluster IP-pool end IP address*
     Description: All nodes for the workload-cluster will be provisioned within this IP pool
-    DefaultValue: ''
+    DefaultValue: '0.0.0.0'
     Configurations:
     - cp1w1
     - cp1w2

From d7e868522518974fae7e256ce2ad30c9d001a0de Mon Sep 17 00:00:00 2001
From: Danny Bessems <danny@bessems.eu>
Date: Tue, 28 Mar 2023 13:49:18 +0200
Subject: [PATCH 05/51] Download workloadcluster helm-charts;Revert
 foldernames;Setup git repositories

---
 .../roles/assets/tasks/containerimages.yml    | 15 ++++++++++---
 ansible/roles/assets/tasks/helm.yml           | 20 ++++++++++++++----
 ansible/roles/assets/tasks/main.yml           |  4 +++-
 ansible/roles/assets/tasks/nodetemplates.yml  |  2 +-
 .../bootstrap/roles/metacluster/tasks/git.yml |  6 +++---
 .../roles/workloadcluster/tasks/gitops.yml    | 21 +++++++++++++++++++
 .../workloadcluster/tasks/nodetemplates.yml   |  2 +-
 ansible/vars/workloadcluster.yml              |  9 ++++++++
 8 files changed, 66 insertions(+), 13 deletions(-)

diff --git a/ansible/roles/assets/tasks/containerimages.yml b/ansible/roles/assets/tasks/containerimages.yml
index 74ef9ed..504f49f 100644
--- a/ansible/roles/assets/tasks/containerimages.yml
+++ b/ansible/roles/assets/tasks/containerimages.yml
@@ -14,22 +14,31 @@
   loop_control:
     label: "{{ item.dest | basename }}"
 
-- name: Parse helm charts for container images
+- name: Parse metacluster helm charts for container images
   ansible.builtin.shell:
     cmd: "{{ item.value.helm.parse_logic }}"
     chdir: /opt/metacluster/helm-charts/{{ item.key }}
-  register: chartimages
+  register: chartimages_metacluster
   when: item.value.helm is defined
   loop: "{{ lookup('ansible.builtin.dict', components) }}"
   loop_control:
     label: "{{ item.key }}"
 
+- name: Parse workloadcluster helm charts for container images
+  ansible.builtin.shell:
+    cmd: "{{ item.value.parse_logic }}"
+    chdir: /opt/workloadcluster/helm-charts/{{ item.key }}
+  register: chartimages_workloadcluster
+  loop: "{{ lookup('ansible.builtin.dict', downstream.helm_charts) }}"
+  loop_control:
+    label: "{{ item.key }}"
+
 - name: Store container images in dicts
   ansible.builtin.set_fact:
     containerimages_{{ item.source }}: "{{ item.results }}"
   loop:
     - source: charts
-      results: "{{ chartimages | json_query('results[*].stdout_lines') | select() | flatten | list }}"
+      results: "{{ (chartimages_metacluster | json_query('results[*].stdout_lines')) + (chartimages_workloadcluster | json_query('results[*].stdout_lines')) | select() | flatten | list }}"
     - source: kubeadm
       results: "{{ kubeadmimages.stdout_lines }}"
     - source: manifests
diff --git a/ansible/roles/assets/tasks/helm.yml b/ansible/roles/assets/tasks/helm.yml
index f03d073..e511453 100644
--- a/ansible/roles/assets/tasks/helm.yml
+++ b/ansible/roles/assets/tasks/helm.yml
@@ -3,17 +3,29 @@
     name: "{{ item.name }}"
     repo_url: "{{ item.url }}"
     state: present
-  loop: "{{ platform.helm_repositories }}"
+  loop: "{{ platform.helm_repositories + downstream.helm_repositories }}"
 
-- name: Fetch helm charts
+- name: Fetch helm charts for metacluster
   ansible.builtin.command:
     cmd: helm fetch {{ item.value.helm.chart }} --untar --version {{ item.value.helm.version }}
     chdir: /opt/metacluster/helm-charts
   when: item.value.helm is defined
-  register: helmcharts
+  register: helmcharts_metacluster
   loop: "{{ lookup('ansible.builtin.dict', components) }}"
   loop_control:
     label: "{{ item.key }}"
   retries: 5
   delay: 5
-  until: helmcharts is not failed
+  until: helmcharts_metacluster is not failed
+
+- name: Fetch helm charts for workloadcluster
+  ansible.builtin.command:
+    cmd: helm fetch {{ item.value.chart }} --untar --version {{ item.value.version }}
+    chdir: /opt/workloadcluster/helm-charts
+  register: helmcharts_workloadcluster
+  loop: "{{ lookup('ansible.builtin.dict', downstream.helm_charts) }}"
+  loop_control:
+    label: "{{ item.key }}"
+  retries: 5
+  delay: 5
+  until: helmcharts_workloadcluster is not failed
diff --git a/ansible/roles/assets/tasks/main.yml b/ansible/roles/assets/tasks/main.yml
index 13d7529..23ba858 100644
--- a/ansible/roles/assets/tasks/main.yml
+++ b/ansible/roles/assets/tasks/main.yml
@@ -16,7 +16,9 @@
     - /opt/metacluster/helm-charts
     - /opt/metacluster/k3s
     - /opt/metacluster/kube-vip
-    - /opt/metacluster/node-templates
+    - /opt/workloadcluster/git-repositories/gitops
+    - /opt/workloadcluster/helm-charts
+    - /opt/workloadcluster/node-templates
     - /var/lib/rancher/k3s/agent/images
     - /var/lib/rancher/k3s/server/manifests
 
diff --git a/ansible/roles/assets/tasks/nodetemplates.yml b/ansible/roles/assets/tasks/nodetemplates.yml
index 52104f2..eb7b93c 100644
--- a/ansible/roles/assets/tasks/nodetemplates.yml
+++ b/ansible/roles/assets/tasks/nodetemplates.yml
@@ -1,4 +1,4 @@
 - name: Download node-template image
   ansible.builtin.uri:
     url: "{{ components.clusterapi.workload.node_template.url }}"
-    dest: /opt/metacluster/node-templates/{{ components.clusterapi.workload.node_template.url | basename}}
+    dest: /opt/workloadcluster/node-templates/{{ components.clusterapi.workload.node_template.url | basename}}
diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/git.yml b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/git.yml
index 10fab3b..669d040 100644
--- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/git.yml
+++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/git.yml
@@ -110,12 +110,12 @@
             - organization: mc
               body:
                 name: GitOps.Config
-                auto_init: true
-                default_branch: main
+                # auto_init: true
+                # default_branch: main
                 description: GitOps manifests
             - organization: wl
               body:
-                name: Template.GitOps.Config
+                name: GitOps.Config
                 # auto_init: true
                 # default_branch: main
                 description: GitOps manifests
diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/gitops.yml b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/gitops.yml
index 1112f4f..7355580 100644
--- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/gitops.yml
+++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/gitops.yml
@@ -1,3 +1,24 @@
+- name: Create hard-links to populate new git-repository
+  ansible.builtin.shell:
+    cmd: >-
+      cp -lr /opt/metacluster/helm-charts/{{ item.key }}/ /opt/workloadcluster/git-repositories/gitops/
+  loop: "{{ lookup('ansible.builtin.dict', downstream.helm_charts) }}"
+  loop_control:
+    label: "{{ item.key }}"
+
+- name: Initialize/Push git repository
+  ansible.builtin.shell:
+    cmd: |
+      git init
+      git config --global user.email "administrator@{{ vapp['metacluster.fqdn'] }}"
+      git config --global user.name "administrator"
+      git checkout -b main
+      git add .
+      git commit -m "Upload charts"
+      git remote add origin https://git.{{ vapp['metacluster.fqdn'] }}/wl/GitOps.Config.git
+      git push https://administrator:{{ vapp['metacluster.password'] | urlencode }}@git.{{ vapp['metacluster.fqdn'] }}/wl/GitOps.Config.git --all
+    chdir: /opt/workloadcluster/git-repositories/gitops
+
 - block:
 
     - name: Generate service account in workload cluster
diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/nodetemplates.yml b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/nodetemplates.yml
index e867b67..c002cb5 100644
--- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/nodetemplates.yml
+++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/nodetemplates.yml
@@ -62,7 +62,7 @@
       when: existing_ova is failed
 
   vars:
-    filename: "{{ query('ansible.builtin.fileglob', '/opt/metacluster/node-templates/*.ova') | first }}"
+    filename: "{{ query('ansible.builtin.fileglob', '/opt/workloadcluster/node-templates/*.ova') | first }}"
   module_defaults:
     group/vmware:
       hostname: "{{ vapp['hv.fqdn'] }}"
diff --git a/ansible/vars/workloadcluster.yml b/ansible/vars/workloadcluster.yml
index ae82752..a64dcea 100644
--- a/ansible/vars/workloadcluster.yml
+++ b/ansible/vars/workloadcluster.yml
@@ -3,6 +3,8 @@ downstream:
   helm_repositories:
     - name: longhorn
       url: https://charts.longhorn.io
+    - name: sealed-secrets
+      url: https://bitnami-labs.github.io/sealed-secrets
 
   helm_charts:
 
@@ -13,3 +15,10 @@ downstream:
       chart_values: !unsafe |
         defaultSettings:
           defaultDataPath: /mnt/blockstorage
+
+    sealedsecrets:
+      version: 2.8.1  # (= Sealed Secrets v0.20.2)
+      chart: sealed-secrets/sealed-secrets
+      parse_logic: helm template . | yq --no-doc eval '.. | .image? | select(.)' | sort -u | awk '!/ /'
+      chart_values: !unsafe |
+        # Empty

From cd5fa89a0db40b833ca572c5e4cd68099fc20fb9 Mon Sep 17 00:00:00 2001
From: Danny Bessems <danny@bessems.eu>
Date: Tue, 28 Mar 2023 16:41:48 +0200
Subject: [PATCH 06/51] Fix key reference

---
 ansible/vars/workloadcluster.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ansible/vars/workloadcluster.yml b/ansible/vars/workloadcluster.yml
index a64dcea..230fcce 100644
--- a/ansible/vars/workloadcluster.yml
+++ b/ansible/vars/workloadcluster.yml
@@ -16,7 +16,7 @@ downstream:
         defaultSettings:
           defaultDataPath: /mnt/blockstorage
 
-    sealedsecrets:
+    sealed-secrets:
       version: 2.8.1  # (= Sealed Secrets v0.20.2)
       chart: sealed-secrets/sealed-secrets
       parse_logic: helm template . | yq --no-doc eval '.. | .image? | select(.)' | sort -u | awk '!/ /'

From eb2f491f724b33d4817138caff55ee91582fcdc2 Mon Sep 17 00:00:00 2001
From: Danny Bessems <danny@bessems.eu>
Date: Tue, 28 Mar 2023 22:31:11 +0200
Subject: [PATCH 07/51] Refactor git repo creation;Housekeeping

---
 ansible/roles/assets/tasks/manifests.yml            | 12 ++++++++++--
 .../roles/workloadcluster/tasks/gitops.yml          | 13 ++++++++++---
 .../roles/workloadcluster/tasks/nodetemplates.yml   |  2 +-
 3 files changed, 21 insertions(+), 6 deletions(-)

diff --git a/ansible/roles/assets/tasks/manifests.yml b/ansible/roles/assets/tasks/manifests.yml
index adf2094..0fa58b4 100644
--- a/ansible/roles/assets/tasks/manifests.yml
+++ b/ansible/roles/assets/tasks/manifests.yml
@@ -2,19 +2,27 @@
 
     - name: Aggregate chart_values into dict
       ansible.builtin.set_fact:
-        chart_values: "{{ chart_values | default({}) | combine({ (item.key | regex_replace('[^A-Za-z0-9]', '')): { 'chart_values': (item.value.helm.chart_values | from_yaml) } }) }}"
+        metacluster_chartvalues: "{{ metacluster_chartvalues | default({}) | combine({ (item.key | regex_replace('[^A-Za-z0-9]', '')): { 'chart_values': (item.value.helm.chart_values | from_yaml) } }) }}"
       when: item.value.helm.chart_values is defined
       loop: "{{ lookup('ansible.builtin.dict', components) }}"
       loop_control:
         label: "{{ item.key }}"
 
+    # - name: Aggregate chart_values into dict
+    #   ansible.builtin.set_fact:
+    #     workloadcluster_chartvalues: "{{ workloadcluster_chartvalues | default({}) | combine({ (item.key | regex_replace('[^A-Za-z0-9]', '')): { 'chart_values': (item.value.chart_values | from_yaml) } }) }}"
+    #   when: item.value.chart_values is defined
+    #   loop: "{{ lookup('ansible.builtin.dict', downstream.helm_charts) }}"
+    #   loop_control:
+    #     label: "{{ item.key }}"
+
     - name: Write dict to vars_file
       ansible.builtin.copy:
         dest: /opt/firstboot/ansible/vars/metacluster.yml
         content: >-
           {{
             { 'components': (
-              chart_values |
+              metacluster_chartvalues |
               combine({ 'clusterapi': components.clusterapi }) |
               combine({ 'kubevip'   : components.kubevip }) )
             } | to_nice_yaml(indent=2, width=4096)
diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/gitops.yml b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/gitops.yml
index 7355580..77147a0 100644
--- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/gitops.yml
+++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/gitops.yml
@@ -1,10 +1,17 @@
+- name: Aggregate helm charts from filesystem
+  ansible.builtin.find:
+    path: /opt/workloadcluster/helm-charts
+    file_type: directory
+    recurse: false
+  register: helm_charts
+
 - name: Create hard-links to populate new git-repository
   ansible.builtin.shell:
     cmd: >-
-      cp -lr /opt/metacluster/helm-charts/{{ item.key }}/ /opt/workloadcluster/git-repositories/gitops/
-  loop: "{{ lookup('ansible.builtin.dict', downstream.helm_charts) }}"
+      cp -lr {{ item.path }}/ /opt/workloadcluster/git-repositories/gitops/
+  loop: "{{ helm_charts.files }}"
   loop_control:
-    label: "{{ item.key }}"
+    label: "{{ item.path | basename }}"
 
 - name: Initialize/Push git repository
   ansible.builtin.shell:
diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/nodetemplates.yml b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/nodetemplates.yml
index c002cb5..783ef2c 100644
--- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/nodetemplates.yml
+++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/nodetemplates.yml
@@ -38,7 +38,7 @@
           community.vmware.vmware_guest_disk:
             name: "{{ ova_deploy.instance.hw_name }}"
             disk:
-              - size: 1Gb
+              - size: 1Mb
                 scsi_controller: 1
                 scsi_type: paravirtual
                 unit_number: 0

From 767be3b8f5549eb147bd217e64fb1d595196012c Mon Sep 17 00:00:00 2001
From: Danny Bessems <danny@bessems.eu>
Date: Tue, 28 Mar 2023 22:36:21 +0200
Subject: [PATCH 08/51] Upgrade CAPI/CAPV

---
 ansible/vars/metacluster.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/ansible/vars/metacluster.yml b/ansible/vars/metacluster.yml
index e9b7a0f..7da1b70 100644
--- a/ansible/vars/metacluster.yml
+++ b/ansible/vars/metacluster.yml
@@ -85,10 +85,10 @@ components:
     management:
       version:
         # Must match the version referenced at `dependencies.static_binaries[.filename==clusterctl].url`
-        base: v1.3.5
+        base: v1.4.0
         # Must match the version referenced at `components.cert-manager.helm.version`
         cert_manager: v1.11.0
-        infrastructure_vsphere: v1.5.3
+        infrastructure_vsphere: v1.6.0
         ipam_incluster: v0.1.0-alpha.2
     workload:
       version:
@@ -276,7 +276,7 @@ dependencies:
 
   static_binaries:
     - filename: clusterctl
-      url: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.3.5/clusterctl-linux-amd64
+      url: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.4.0/clusterctl-linux-amd64
     - filename: govc
       url: https://github.com/vmware/govmomi/releases/download/v0.29.0/govc_Linux_x86_64.tar.gz
       archive: compressed

From 5effe00c19394fc20c87fe436db58f7009cdc140 Mon Sep 17 00:00:00 2001
From: Danny Bessems <danny@bessems.eu>
Date: Wed, 29 Mar 2023 13:53:56 +0200
Subject: [PATCH 09/51] Upgrade version

---
 ansible/vars/metacluster.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ansible/vars/metacluster.yml b/ansible/vars/metacluster.yml
index 7da1b70..77a652e 100644
--- a/ansible/vars/metacluster.yml
+++ b/ansible/vars/metacluster.yml
@@ -58,7 +58,7 @@ components:
 
   argo-cd:
     helm:
-      version: 5.24.0  # (= ArgoCD v2.6.3)
+      version: 5.27.4  # (= ArgoCD v2.6.7)
       chart: argo/argo-cd
       parse_logic: helm template . | yq --no-doc eval '.. | .image? | select(.)' | sort -u | awk '!/ /'
       chart_values: !unsafe |

From 29396de154abc24c858916b811021d5f4defbb65 Mon Sep 17 00:00:00 2001
From: Danny Bessems <danny@bessems.eu>
Date: Wed, 29 Mar 2023 22:01:43 +0200
Subject: [PATCH 10/51] Inject CPI image tag into manifest

---
 ansible/roles/assets/tasks/manifests.yml | 6 ++++++
 ansible/vars/metacluster.yml             | 2 ++
 2 files changed, 8 insertions(+)

diff --git a/ansible/roles/assets/tasks/manifests.yml b/ansible/roles/assets/tasks/manifests.yml
index 0fa58b4..2348c44 100644
--- a/ansible/roles/assets/tasks/manifests.yml
+++ b/ansible/roles/assets/tasks/manifests.yml
@@ -73,6 +73,12 @@
   delay: 5
   until: clusterapi_manifests is not failed
 
+- name: Update cluster-template with image tags
+  ansible.builtin.replace:
+    dest: infrastructure-vsphere/{{ components.clusterapi.management.version.infrastructure_vsphere }}/cluster-template.yaml
+    regexp: ':\${CPI_IMAGE_K8S_VERSION}'
+    replace: ":{{ components.clusterapi.management.version.cpi_vsphere }}"
+
 - name: Download kube-vip RBAC manifest
   ansible.builtin.get_url:
     url: https://kube-vip.io/manifests/rbac.yaml
diff --git a/ansible/vars/metacluster.yml b/ansible/vars/metacluster.yml
index 77a652e..1862fc7 100644
--- a/ansible/vars/metacluster.yml
+++ b/ansible/vars/metacluster.yml
@@ -90,6 +90,8 @@ components:
         cert_manager: v1.11.0
         infrastructure_vsphere: v1.6.0
         ipam_incluster: v0.1.0-alpha.2
+        # Refer to `https://console.cloud.google.com/gcr/images/cloud-provider-vsphere/GLOBAL/cpi/release/manager` for available tags
+        cpi_vsphere: v1.26.0
     workload:
       version:
         calico: v3.25.0

From 414b72bcb8554d6c841ed63842f25de56f4eee37 Mon Sep 17 00:00:00 2001
From: Danny Bessems <danny@bessems.eu>
Date: Wed, 29 Mar 2023 22:53:56 +0200
Subject: [PATCH 11/51] Fix path

---
 ansible/roles/assets/tasks/manifests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ansible/roles/assets/tasks/manifests.yml b/ansible/roles/assets/tasks/manifests.yml
index 2348c44..74240a7 100644
--- a/ansible/roles/assets/tasks/manifests.yml
+++ b/ansible/roles/assets/tasks/manifests.yml
@@ -75,7 +75,7 @@
 
 - name: Update cluster-template with image tags
   ansible.builtin.replace:
-    dest: infrastructure-vsphere/{{ components.clusterapi.management.version.infrastructure_vsphere }}/cluster-template.yaml
+    dest: /opt/metacluster/cluster-api/infrastructure-vsphere/{{ components.clusterapi.management.version.infrastructure_vsphere }}/cluster-template.yaml
     regexp: ':\${CPI_IMAGE_K8S_VERSION}'
     replace: ":{{ components.clusterapi.management.version.cpi_vsphere }}"
 

From 1c60214f5ad1b080bbc5365e9f10fbab88556f6c Mon Sep 17 00:00:00 2001
From: Danny Bessems <danny@bessems.eu>
Date: Fri, 31 Mar 2023 14:25:03 +0200
Subject: [PATCH 12/51] Add repositories;Push manifests;Change protocol

---
 .../bootstrap/roles/metacluster/tasks/git.yml       | 10 ++++++++--
 .../roles/workloadcluster/tasks/clusterapi.yml      | 13 +++++++++++++
 .../bootstrap/templates/applicationset.j2           |  4 ++--
 3 files changed, 23 insertions(+), 4 deletions(-)

diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/git.yml b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/git.yml
index 669d040..34aae1b 100644
--- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/git.yml
+++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/git.yml
@@ -109,10 +109,16 @@
           loop:
             - organization: mc
               body:
-                name: GitOps.Config
+                name: GitOps.HelmCharts
                 # auto_init: true
                 # default_branch: main
-                description: GitOps manifests
+                description: Helm charts
+            - organization: mc
+              body:
+                name: GitOps.ClusterAPI
+                # auto_init: true
+                # default_branch: main
+                description: ClusterAPI manifests
             - organization: wl
               body:
                 name: GitOps.Config
diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/clusterapi.yml b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/clusterapi.yml
index a6bea8f..10429ef 100644
--- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/clusterapi.yml
+++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/clusterapi.yml
@@ -154,6 +154,19 @@
         -f {{ capi_clustermanifest.path }} \
         -o /opt/metacluster/cluster-api/new-cluster
 
+- name: Initialize/Push git repository
+  ansible.builtin.shell:
+    cmd: |
+      git init
+      git config --global user.email "administrator@{{ vapp['metacluster.fqdn'] }}"
+      git config --global user.name "administrator"
+      git checkout -b main
+      git add .
+      git commit -m "Upload manifests"
+      git remote add origin https://git.{{ vapp['metacluster.fqdn'] }}/mc/GitOps.ClusterAPI.git
+      git push https://administrator:{{ vapp['metacluster.password'] | urlencode }}@git.{{ vapp['metacluster.fqdn'] }}/mc/GitOps.ClusterAPI.git --all
+    chdir: /opt/metacluster/cluster-api/new-cluster
+
 - name: Cleanup tempfile
   ansible.builtin.file:
     path: "{{ capi_clustermanifest.path }}"
diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/applicationset.j2 b/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/applicationset.j2
index 63303a8..0f8bf57 100644
--- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/applicationset.j2
+++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/applicationset.j2
@@ -6,7 +6,7 @@ metadata:
 spec:
   generators:
   - git:
-      repoURL: ssh://git@gitea-ssh.gitea.svc.cluster.local/mc/GitOps.Config.git
+      repoURL: https://git.{{ vapp['metacluster.fqdn'] }}/wl/GitOps.Config.git
       revision: HEAD
       directories:
       - path: metacluster-applicationset/*
@@ -20,7 +20,7 @@ spec:
           prune: true
           selfHeal: true
       source:
-        repoURL: ssh://git@gitea-ssh.gitea.svc.cluster.local/mc/GitOps.Config.git
+        repoURL: https://git.{{ vapp['metacluster.fqdn'] }}/wl/GitOps.Config.git
         targetRevision: HEAD
         path: {% raw %}'{{ path }}'{% endraw +%}
       destination:

From 933615adebb71bf2f0304d8a702129fd3eb75f9c Mon Sep 17 00:00:00 2001
From: Danny Bessems <danny@bessems.eu>
Date: Fri, 31 Mar 2023 18:19:13 +0200
Subject: [PATCH 13/51] Refactor gitops repositories;Move capi manifests to
 subfolder;Sort components in tty console message;Generalize templates

---
 .../roles/metacluster/tasks/gitops.yml        | 13 +++++--
 .../workloadcluster/tasks/clusterapi.yml      | 21 +++++-----
 .../roles/workloadcluster/tasks/gitops.yml    | 39 +++++++++++++++++++
 .../bootstrap/templates/applicationset.j2     | 12 +++---
 .../bootstrap/templates/gitrepo.j2            |  4 +-
 .../ansible_payload/common/templates/tty.j2   |  2 +-
 6 files changed, 68 insertions(+), 23 deletions(-)

diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/gitops.yml b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/gitops.yml
index 865b300..b4b1b1c 100644
--- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/gitops.yml
+++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/gitops.yml
@@ -41,7 +41,7 @@
         _template:
           name: argocd-gitrepo-metacluster
           namespace: argo-cd
-          uid: "{{ lookup('ansible.builtin.password', '/dev/null length=5 chars=ascii_lowercase,digits seed=inventory_hostname') }}"
+          url: https://git.{{ vapp['metacluster.fqdn'] }}/mc/GitOps.Config.git
           privatekey: "{{ lookup('ansible.builtin.file', '~/.ssh/git_rsa_id') | indent(4, true) }}"
       notify:
         - Apply manifests
@@ -49,14 +49,19 @@
     - name: Create applicationset
       ansible.builtin.template:
         src: applicationset.j2
-        dest: /var/lib/rancher/k3s/server/manifests/{{ _template.name }}-manifest.yaml
+        dest: /var/lib/rancher/k3s/server/manifests/{{ _template.application.name }}-manifest.yaml
         owner: root
         group: root
         mode: 0600
       vars:
         _template:
-          name: argocd-applicationset-metacluster
-          namespace: argo-cd
+          application:
+            name: argocd-applicationset-metacluster
+            namespace: argo-cd
+          cluster:
+            name: https://kubernetes.default.svc
+          repository:
+            url: https://git.{{ vapp['metacluster.fqdn'] }}/mc/GitOps.Config.git
       notify:
         - Apply manifests
 
diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/clusterapi.yml b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/clusterapi.yml
index 10429ef..5ed2c42 100644
--- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/clusterapi.yml
+++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/clusterapi.yml
@@ -137,9 +137,9 @@
     chdir: /opt/metacluster/cluster-api
   register: clusterctl_newcluster
 
-- name: Initialize tempfile
+- name: Initialize tempfolder
   ansible.builtin.tempfile:
-    state: file
+    state: directory
   register: capi_clustermanifest
 
 - name: Save workload cluster manifest
@@ -152,7 +152,7 @@
     cmd: >-
       kubectl slice \
         -f {{ capi_clustermanifest.path }} \
-        -o /opt/metacluster/cluster-api/new-cluster
+        -o {{ capi_clustermanifest.path }}/manifests
 
 - name: Initialize/Push git repository
   ansible.builtin.shell:
@@ -161,17 +161,18 @@
       git config --global user.email "administrator@{{ vapp['metacluster.fqdn'] }}"
       git config --global user.name "administrator"
       git checkout -b main
-      git add .
+      git add ./manifests
       git commit -m "Upload manifests"
       git remote add origin https://git.{{ vapp['metacluster.fqdn'] }}/mc/GitOps.ClusterAPI.git
       git push https://administrator:{{ vapp['metacluster.password'] | urlencode }}@git.{{ vapp['metacluster.fqdn'] }}/mc/GitOps.ClusterAPI.git --all
-    chdir: /opt/metacluster/cluster-api/new-cluster
+    chdir: "{{ capi_clustermanifest.path }}"
 
-- name: Cleanup tempfile
-  ansible.builtin.file:
-    path: "{{ capi_clustermanifest.path }}"
-    state: absent
-  when: capi_clustermanifest.path is defined
+# Temporarily disabled until manifests are properly managed by gitops
+# - name: Cleanup tempfolder
+#   ansible.builtin.file:
+#     path: "{{ capi_clustermanifest.path }}"
+#     state: absent
+#   when: capi_clustermanifest.path is defined
 
 - name: Create in-cluster IpPool
   kubernetes.core.k8s:
diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/gitops.yml b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/gitops.yml
index 77147a0..961fe51 100644
--- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/gitops.yml
+++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/gitops.yml
@@ -53,6 +53,45 @@
             url: https://{{ vapp['workloadcluster.vip'] }}:6443
             token: "{{ workloadcluster_bearertoken.resources | json_query('[].data.token') }}"
 
+    - name: Configure workload-cluster GitOps repository
+      ansible.builtin.template:
+        src: gitrepo.j2
+        dest: /var/lib/rancher/k3s/server/manifests/{{ _template.name }}-manifest.yaml
+        owner: root
+        group: root
+        mode: 0600
+      vars:
+        _template:
+          name: argocd-gitrepo-workloadcluster
+          namespace: argo-cd
+          url: https://git.{{ vapp['metacluster.fqdn'] }}/wl/GitOps.Config.git
+          privatekey: "{{ lookup('ansible.builtin.file', '~/.ssh/git_rsa_id') | indent(4, true) }}"
+      notify:
+        - Apply manifests
+
+    - name: Create applicationset
+      ansible.builtin.template:
+        src: applicationset.j2
+        dest: /var/lib/rancher/k3s/server/manifests/{{ _template.application.name }}-manifest.yaml
+        owner: root
+        group: root
+        mode: 0600
+      vars:
+        _template:
+          application:
+            name: argocd-applicationset-workloadcluster
+            namespace: argo-cd
+          cluster:
+            name: "{{ vapp['workloadcluster.name'] | lower }}"
+          repository:
+            url: https://git.{{ vapp['metacluster.fqdn'] }}/wl/GitOps.Config.git
+
+      notify:
+        - Apply manifests
+
+    - name: Trigger handlers
+      ansible.builtin.meta: flush_handlers
+
   vars:
     _template:
       account:
diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/applicationset.j2 b/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/applicationset.j2
index 0f8bf57..5417227 100644
--- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/applicationset.j2
+++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/applicationset.j2
@@ -1,15 +1,15 @@
 apiVersion: argoproj.io/v1alpha1
 kind: ApplicationSet
 metadata:
-  name: {{ _template.name }}
-  namespace: {{ _template.namespace }}
+  name: {{ _template.application.name }}
+  namespace: {{ _template.application.namespace }}
 spec:
   generators:
   - git:
-      repoURL: https://git.{{ vapp['metacluster.fqdn'] }}/wl/GitOps.Config.git
+      repoURL: {{ _template.repository.url }}
       revision: HEAD
       directories:
-      - path: metacluster-applicationset/*
+      - path: applicationset/*
   template:
     metadata:
       name: {% raw %}'{{ path.basename }}'{% endraw +%}
@@ -20,9 +20,9 @@ spec:
           prune: true
           selfHeal: true
       source:
-        repoURL: https://git.{{ vapp['metacluster.fqdn'] }}/wl/GitOps.Config.git
+        repoURL: {{ _template.repository.url }}
         targetRevision: HEAD
         path: {% raw %}'{{ path }}'{% endraw +%}
       destination:
-        server: https://kubernetes.default.svc
+        server: {{ _template.cluster.name }}
         namespace: default
diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/gitrepo.j2 b/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/gitrepo.j2
index c5351f9..6541588 100644
--- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/gitrepo.j2
+++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/gitrepo.j2
@@ -1,12 +1,12 @@
 apiVersion: v1
 kind: Secret
 metadata:
-  name: {{ _template.name }}-{{ _template.uid }}
+  name: {{ _template.name }}
   namespace: {{ _template.namespace }}
   labels:
     argocd.argoproj.io/secret-type: repository
 stringData:
-  url: ssh://git@gitea-ssh.gitea.svc.cluster.local/mc/GitOps.Config.git
+  url: {{ _template.url }}
   name: {{ _template.name }}
   insecure: 'true'
   sshPrivateKey: |
diff --git a/ansible/roles/firstboot/files/ansible_payload/common/templates/tty.j2 b/ansible/roles/firstboot/files/ansible_payload/common/templates/tty.j2
index 8eb0e83..882376d 100644
--- a/ansible/roles/firstboot/files/ansible_payload/common/templates/tty.j2
+++ b/ansible/roles/firstboot/files/ansible_payload/common/templates/tty.j2
@@ -33,7 +33,7 @@ while /bin/true; do
   echo -e "${PRST}" > /dev/tty1
   echo -e "\n\n\t${DFLT}To manage this appliance, please connect to one of the following:${LCLR}\n" > /dev/tty1
 
-  for c in "${!COMPONENTS[@]}"; do
+  for c in $( echo "${!COMPONENTS[@]}" | tr ' ' $'\n' | sort); do
     STATUS=$(curl -kLs "${COMPONENTS[${c}]}" -o /dev/null -w '%{http_code}')
 
     if [[ "${STATUS}" -eq "200" ]]; then

From d187f60091a48be3c97fc8c89f89971e6b8f9774 Mon Sep 17 00:00:00 2001
From: Danny Bessems <danny@bessems.eu>
Date: Fri, 31 Mar 2023 18:27:45 +0200
Subject: [PATCH 14/51] Remove redundant key

---
 .../bootstrap/roles/metacluster/tasks/gitops.yml              | 1 -
 .../bootstrap/roles/workloadcluster/tasks/gitops.yml          | 1 -
 .../files/ansible_payload/bootstrap/templates/gitrepo.j2      | 4 ----
 3 files changed, 6 deletions(-)

diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/gitops.yml b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/gitops.yml
index b4b1b1c..61944ef 100644
--- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/gitops.yml
+++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/gitops.yml
@@ -42,7 +42,6 @@
           name: argocd-gitrepo-metacluster
           namespace: argo-cd
           url: https://git.{{ vapp['metacluster.fqdn'] }}/mc/GitOps.Config.git
-          privatekey: "{{ lookup('ansible.builtin.file', '~/.ssh/git_rsa_id') | indent(4, true) }}"
       notify:
         - Apply manifests
 
diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/gitops.yml b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/gitops.yml
index 961fe51..5adcd07 100644
--- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/gitops.yml
+++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/gitops.yml
@@ -65,7 +65,6 @@
           name: argocd-gitrepo-workloadcluster
           namespace: argo-cd
           url: https://git.{{ vapp['metacluster.fqdn'] }}/wl/GitOps.Config.git
-          privatekey: "{{ lookup('ansible.builtin.file', '~/.ssh/git_rsa_id') | indent(4, true) }}"
       notify:
         - Apply manifests
 
diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/gitrepo.j2 b/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/gitrepo.j2
index 6541588..643e1e7 100644
--- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/gitrepo.j2
+++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/gitrepo.j2
@@ -7,7 +7,3 @@ metadata:
     argocd.argoproj.io/secret-type: repository
 stringData:
   url: {{ _template.url }}
-  name: {{ _template.name }}
-  insecure: 'true'
-  sshPrivateKey: |
-{{ _template.privatekey }}

From 570047df3b4035d6ca833712803e38f6a75b1a61 Mon Sep 17 00:00:00 2001
From: Danny Bessems <danny@bessems.eu>
Date: Sat, 1 Apr 2023 13:43:36 +0200
Subject: [PATCH 15/51] Fix target paths;Add git repositories

---
 .../bootstrap/roles/metacluster/tasks/git.yml  | 18 ++++++++++++------
 .../roles/workloadcluster/tasks/clusterapi.yml |  4 ++--
 2 files changed, 14 insertions(+), 8 deletions(-)

diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/git.yml b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/git.yml
index 34aae1b..2b2dfe6 100644
--- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/git.yml
+++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/git.yml
@@ -107,18 +107,24 @@
               Authorization: token {{ gitea_api_token.json.sha1 }}
             body: "{{ item.body }}"
           loop:
-            - organization: mc
-              body:
-                name: GitOps.HelmCharts
-                # auto_init: true
-                # default_branch: main
-                description: Helm charts
             - organization: mc
               body:
                 name: GitOps.ClusterAPI
                 # auto_init: true
                 # default_branch: main
                 description: ClusterAPI manifests
+            - organization: mc
+              body:
+                name: GitOps.Config
+                # auto_init: true
+                # default_branch: main
+                description: GitOps manifests
+            - organization: mc
+              body:
+                name: GitOps.HelmCharts
+                # auto_init: true
+                # default_branch: main
+                description: Helm charts
             - organization: wl
               body:
                 name: GitOps.Config
diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/clusterapi.yml b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/clusterapi.yml
index 5ed2c42..1196177 100644
--- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/clusterapi.yml
+++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/clusterapi.yml
@@ -144,14 +144,14 @@
 
 - name: Save workload cluster manifest
   ansible.builtin.copy:
-    dest: "{{ capi_clustermanifest.path }}"
+    dest: "{{ capi_clustermanifest.path }}/new-cluster.yaml"
     content: "{{ clusterctl_newcluster.stdout }}"
 
 - name: Split manifest into separate files
   ansible.builtin.shell:
     cmd: >-
       kubectl slice \
-        -f {{ capi_clustermanifest.path }} \
+        -f {{ capi_clustermanifest.path }}/new-cluster.yaml \
         -o {{ capi_clustermanifest.path }}/manifests
 
 - name: Initialize/Push git repository

From 3f9fc4b7aa3912706deed2b510bab2750c9cff5a Mon Sep 17 00:00:00 2001
From: Danny Bessems <danny@bessems.eu>
Date: Sat, 1 Apr 2023 16:10:38 +0200
Subject: [PATCH 16/51] Fix git repository organization;Move cluster api
 manifests to gitops;Rename gitrepo's

---
 .../bootstrap/roles/metacluster/tasks/git.yml | 12 ++---
 .../roles/metacluster/tasks/gitops.yml        |  2 +-
 .../workloadcluster/tasks/clusterapi.yml      | 54 ++++++++++++++++---
 .../roles/workloadcluster/tasks/gitops.yml    |  2 +-
 .../bootstrap/templates/application.j2        | 16 ++++++
 5 files changed, 71 insertions(+), 15 deletions(-)
 create mode 100644 ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/application.j2

diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/git.yml b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/git.yml
index 2b2dfe6..85decc5 100644
--- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/git.yml
+++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/git.yml
@@ -119,18 +119,18 @@
                 # auto_init: true
                 # default_branch: main
                 description: GitOps manifests
-            - organization: mc
-              body:
-                name: GitOps.HelmCharts
-                # auto_init: true
-                # default_branch: main
-                description: Helm charts
             - organization: wl
               body:
                 name: GitOps.Config
                 # auto_init: true
                 # default_branch: main
                 description: GitOps manifests
+            - organization: wl
+              body:
+                name: GitOps.HelmCharts
+                # auto_init: true
+                # default_branch: main
+                description: Helm charts
           loop_control:
             label: "{{ item.organization ~ '/' ~ item.body.name }}"
 
diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/gitops.yml b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/gitops.yml
index 61944ef..915950d 100644
--- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/gitops.yml
+++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/gitops.yml
@@ -39,7 +39,7 @@
         mode: 0600
       vars:
         _template:
-          name: argocd-gitrepo-metacluster
+          name: argocd-gitrepo-mc-gitopsconfig
           namespace: argo-cd
           url: https://git.{{ vapp['metacluster.fqdn'] }}/mc/GitOps.Config.git
       notify:
diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/clusterapi.yml b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/clusterapi.yml
index 1196177..68f92cb 100644
--- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/clusterapi.yml
+++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/clusterapi.yml
@@ -167,6 +167,21 @@
       git push https://administrator:{{ vapp['metacluster.password'] | urlencode }}@git.{{ vapp['metacluster.fqdn'] }}/mc/GitOps.ClusterAPI.git --all
     chdir: "{{ capi_clustermanifest.path }}"
 
+- name: Configure Cluster API repository
+  ansible.builtin.template:
+    src: gitrepo.j2
+    dest: /var/lib/rancher/k3s/server/manifests/{{ _template.name }}-manifest.yaml
+    owner: root
+    group: root
+    mode: 0600
+  vars:
+    _template:
+      name: argocd-gitrepo-mc-gitopsclusterapi
+      namespace: argo-cd
+      url: https://git.{{ vapp['metacluster.fqdn'] }}/mc/GitOps.ClusterAPI.git
+  notify:
+    - Apply manifests
+
 # Temporarily disabled until manifests are properly managed by gitops
 # - name: Cleanup tempfolder
 #   ansible.builtin.file:
@@ -174,6 +189,7 @@
 #     state: absent
 #   when: capi_clustermanifest.path is defined
 
+# TODO: Move to gitops
 - name: Create in-cluster IpPool
   kubernetes.core.k8s:
     template: ippool.j2
@@ -201,13 +217,37 @@
   retries: "{{ playbook.retries }}"
   delay: "{{ (storage_benchmark | int) * (playbook.delay.medium | int) }}"
 
-- name: Apply workload cluster manifest
-  kubernetes.core.k8s:
-    definition: >-
-      {{ clusterctl_newcluster.stdout }}
-    wait: true
-    kubeconfig: "{{ kubeconfig.path }}"
-# TODO: move to git repo
+- name: Create application
+  ansible.builtin.template:
+    src: application.j2
+    dest: /var/lib/rancher/k3s/server/manifests/{{ _template.application.name }}-manifest.yaml
+    owner: root
+    group: root
+    mode: 0600
+  vars:
+    _template:
+      application:
+        name: argocd-application-clusterapi
+        namespace: argo-cd
+      cluster:
+        name: https://kubernetes.default.svc
+        namespace: default
+      repository:
+        url: https://git.{{ vapp['metacluster.fqdn'] }}/mc/GitOps.ClusterAPI.git
+        path: manifests
+        revision: main
+  notify:
+    - Apply manifests
+
+- name: Trigger handlers
+  ansible.builtin.meta: flush_handlers
+
+# - name: Apply workload cluster manifest
+#   kubernetes.core.k8s:
+#     definition: >-
+#       {{ clusterctl_newcluster.stdout }}
+#     wait: true
+#     kubeconfig: "{{ kubeconfig.path }}"
 
 - name: Wait for cluster to be available
   ansible.builtin.shell:
diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/gitops.yml b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/gitops.yml
index 5adcd07..9cf5f65 100644
--- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/gitops.yml
+++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/gitops.yml
@@ -62,7 +62,7 @@
         mode: 0600
       vars:
         _template:
-          name: argocd-gitrepo-workloadcluster
+          name: argocd-gitrepo-wl-gitopsconfig
           namespace: argo-cd
           url: https://git.{{ vapp['metacluster.fqdn'] }}/wl/GitOps.Config.git
       notify:
diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/application.j2 b/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/application.j2
new file mode 100644
index 0000000..828b56a
--- /dev/null
+++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/application.j2
@@ -0,0 +1,16 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: {{ _template.application.name }}
+  namespace: {{ _template.application.namespace }}
+spec:
+  destination:
+    namespace: {{ _template.cluster.namespace }}
+    server: {{ _template.cluster.name }}
+  project: default
+  source:
+    repoURL: {{ _template.repository.url }}
+    path: {{ _template.repository.path }}
+    targetRevision: {{ _template.repository.revision }}
+  syncPolicy:
+    automated: {}

From 89cf69adc7f1be189a8c5df63177eacba3d84347 Mon Sep 17 00:00:00 2001
From: Danny Bessems <danny@bessems.eu>
Date: Sun, 2 Apr 2023 22:36:23 +0200
Subject: [PATCH 17/51] Refactor cluster registration

---
 .../roles/metacluster/tasks/gitops.yml        |  2 +-
 .../roles/workloadcluster/tasks/gitops.yml    | 46 +++++++++++--------
 .../bootstrap/templates/applicationset.j2     |  2 +-
 .../bootstrap/templates/cluster.j2            |  6 ++-
 ...eaccount.j2 => serviceaccount.j2.DISABLED} |  0
 5 files changed, 34 insertions(+), 22 deletions(-)
 rename ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/{serviceaccount.j2 => serviceaccount.j2.DISABLED} (100%)

diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/gitops.yml b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/gitops.yml
index 915950d..4d00ff4 100644
--- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/gitops.yml
+++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/gitops.yml
@@ -58,7 +58,7 @@
             name: argocd-applicationset-metacluster
             namespace: argo-cd
           cluster:
-            name: https://kubernetes.default.svc
+            url: https://kubernetes.default.svc
           repository:
             url: https://git.{{ vapp['metacluster.fqdn'] }}/mc/GitOps.Config.git
       notify:
diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/gitops.yml b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/gitops.yml
index 9cf5f65..1f32468 100644
--- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/gitops.yml
+++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/gitops.yml
@@ -28,19 +28,26 @@
 
 - block:
 
-    - name: Generate service account in workload cluster
-      kubernetes.core.k8s:
-        template: serviceaccount.j2
-        state: present
+    # - name: Generate service account in workload-cluster
+    #   kubernetes.core.k8s:
+    #     template: serviceaccount.j2
+    #     state: present
 
-    - name: Retrieve service account bearer token
+    # - name: Retrieve service account bearer token
+    #   kubernetes.core.k8s_info:
+    #     kind: Secret
+    #     name: "{{ _template.account.name }}-secret"
+    #     namespace: "{{ _template.account.namespace }}"
+    #   register: workloadcluster_bearertoken
+
+    - name: Retrieve workload-cluster kubeconfig
       kubernetes.core.k8s_info:
         kind: Secret
-        name: "{{ _template.account.name }}-secret"
-        namespace: "{{ _template.account.namespace }}"
-      register: workloadcluster_bearertoken
+        name: "{{ vapp['workloadcluster.name'] }}-kubeconfig"
+        namespace: default
+      register: secret_workloadcluster_kubeconfig
 
-    - name: Register workload cluster in argo-cd
+    - name: Register workload-cluster in argo-cd
       kubernetes.core.k8s:
         template: cluster.j2
         state: present
@@ -51,7 +58,10 @@
             name: "{{ vapp['workloadcluster.name'] | lower }}"
             secret: argocd-cluster-{{ vapp['workloadcluster.name'] | lower }}
             url: https://{{ vapp['workloadcluster.vip'] }}:6443
-            token: "{{ workloadcluster_bearertoken.resources | json_query('[].data.token') }}"
+          kubeconfig:
+            ca: (secret_workloadcluster_kubeconfig.resources[0].data.value | b64decode | from_yaml).clusters[0].cluster['certificate-authority-data'] | b64encode
+            certificate: (secret_workloadcluster_kubeconfig.resources[0].data.value | b64decode | from_yaml).users[0].user['client-certificate-data'] | b64encode
+            key: (secret_workloadcluster_kubeconfig.resources[0].data.value | b64decode | from_yaml).users[0].user['client-key-data'] | b64encode
 
     - name: Configure workload-cluster GitOps repository
       ansible.builtin.template:
@@ -81,7 +91,7 @@
             name: argocd-applicationset-workloadcluster
             namespace: argo-cd
           cluster:
-            name: "{{ vapp['workloadcluster.name'] | lower }}"
+            url: https://{{ vapp['workloadcluster.vip'] }}:6443
           repository:
             url: https://git.{{ vapp['metacluster.fqdn'] }}/wl/GitOps.Config.git
 
@@ -91,13 +101,13 @@
     - name: Trigger handlers
       ansible.builtin.meta: flush_handlers
 
-  vars:
-    _template:
-      account:
-        name: argocd-sa
-        namespace: default
-      clusterrolebinding:
-        name: argocd-crb
+  # vars:
+  #   _template:
+  #     account:
+  #       name: argocd-sa
+  #       namespace: default
+  #     clusterrolebinding:
+  #       name: argocd-crb
   module_defaults:
     group/k8s:
       kubeconfig: "{{ capi_kubeconfig.path }}"
diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/applicationset.j2 b/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/applicationset.j2
index 5417227..4160fe4 100644
--- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/applicationset.j2
+++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/applicationset.j2
@@ -24,5 +24,5 @@ spec:
         targetRevision: HEAD
         path: {% raw %}'{{ path }}'{% endraw +%}
       destination:
-        server: {{ _template.cluster.name }}
+        server: {{ _template.cluster.url }}
         namespace: default
diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/cluster.j2 b/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/cluster.j2
index bbaec5c..0d932a8 100644
--- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/cluster.j2
+++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/cluster.j2
@@ -11,8 +11,10 @@ stringData:
   server: {{ _template.cluster.url }}
   config: |
     {
-      "bearerToken": "{{ _template.cluster.token }}",
       "tlsClientConfig": {
-        "insecure": true
+        "insecure": false,
+        "caData": "{{ _template.kubeconfig.ca }}",
+        "certData": "{{ _template.kubeconfig.certificate }}",
+        "keyData": "{{ _template.kubeconfig.key }}"
       }
     }
diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/serviceaccount.j2 b/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/serviceaccount.j2.DISABLED
similarity index 100%
rename from ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/serviceaccount.j2
rename to ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/serviceaccount.j2.DISABLED

From 25230fdda226b82ba5b6bb5c1f1deb9f115df762 Mon Sep 17 00:00:00 2001
From: Danny Bessems <danny@bessems.eu>
Date: Mon, 3 Apr 2023 14:50:58 +0200
Subject: [PATCH 18/51] Avoid parallel build issues

---
 .drone.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.drone.yml b/.drone.yml
index 87b2cc8..b976e12 100644
--- a/.drone.yml
+++ b/.drone.yml
@@ -52,6 +52,7 @@ steps:
   - |
     packer build \
       -on-error=cleanup -timestamp-ui \
+      -parallel=false \
       -var vm_name=$DRONE_BUILD_NUMBER-${DRONE_COMMIT_SHA:0:10} \
       -var docker_username=$${DOCKER_USERNAME} \
       -var docker_password=$${DOCKER_PASSWORD} \

From c054c76b60c2e3b5537df150b760670be5285c9b Mon Sep 17 00:00:00 2001
From: Danny Bessems <danny@bessems.eu>
Date: Mon, 3 Apr 2023 15:00:06 +0200
Subject: [PATCH 19/51] Avoid parallel build issues #2

---
 .drone.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.drone.yml b/.drone.yml
index b976e12..8cc1ee2 100644
--- a/.drone.yml
+++ b/.drone.yml
@@ -52,7 +52,7 @@ steps:
   - |
     packer build \
       -on-error=cleanup -timestamp-ui \
-      -parallel=false \
+      -parallel-builds=1 \
       -var vm_name=$DRONE_BUILD_NUMBER-${DRONE_COMMIT_SHA:0:10} \
       -var docker_username=$${DOCKER_USERNAME} \
       -var docker_password=$${DOCKER_PASSWORD} \

From 7ac4cc0914b79e4c1b47d6e2f5a9f2728959f7e7 Mon Sep 17 00:00:00 2001
From: Danny Bessems <danny@bessems.eu>
Date: Mon, 3 Apr 2023 15:43:10 +0200
Subject: [PATCH 20/51] Avoid parallel build issues #3

---
 .drone.yml | 75 +++++++++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 74 insertions(+), 1 deletion(-)

diff --git a/.drone.yml b/.drone.yml
index 8cc1ee2..4877337 100644
--- a/.drone.yml
+++ b/.drone.yml
@@ -18,7 +18,10 @@ steps:
   - ovftool --version
   - packer --version
   - yamllint --version
+
 - name: Kubernetes Bootstrap Appliance
+  depends_on:
+  - Debugging information
   image: bv11-cr01.bessems.eu/library/packer-extended
   pull: always
   commands:
@@ -52,7 +55,7 @@ steps:
   - |
     packer build \
       -on-error=cleanup -timestamp-ui \
-      -parallel-builds=1 \
+      -only=vsphere-iso.bootstrap \
       -var vm_name=$DRONE_BUILD_NUMBER-${DRONE_COMMIT_SHA:0:10} \
       -var docker_username=$${DOCKER_USERNAME} \
       -var docker_password=$${DOCKER_PASSWORD} \
@@ -81,7 +84,77 @@ steps:
     path: /output
   - name: scratch
     path: /scratch
+
+- name: Kubernetes Upgrade Appliance
+  depends_on:
+  - Debugging information
+  image: bv11-cr01.bessems.eu/library/packer-extended
+  pull: always
+  commands:
+  - |
+    sed -i -e "s/<<img-password>>/$${SSH_PASSWORD}/g" \
+      packer/preseed/UbuntuServer22.04/user-data
+  - |
+    export K8S_VERSION=$(yq '.components.clusterapi.workload.version.k8s' < ./ansible/vars/metacluster.yml)
+  - |
+    yamllint -d "{extends: relaxed, rules: {line-length: disable}}" \
+      ansible \
+      packer/preseed/UbuntuServer22.04/user-data \
+      scripts
+  - |
+    ansible-galaxy install \
+      -r ansible/requirements.yml
+  - |
+    packer init -upgrade \
+      ./packer
+  - |
+    packer validate \
+      -var vm_name=$DRONE_BUILD_NUMBER-${DRONE_COMMIT_SHA:0:10} \
+      -var docker_username=$${DOCKER_USERNAME} \
+      -var docker_password=$${DOCKER_PASSWORD} \
+      -var repo_username=$${REPO_USERNAME} \
+      -var repo_password=$${REPO_PASSWORD} \
+      -var ssh_password=$${SSH_PASSWORD} \
+      -var vsphere_password=$${VSPHERE_PASSWORD} \
+      -var k8s_version=$K8S_VERSION \
+      ./packer
+  - |
+    packer build \
+      -on-error=cleanup -timestamp-ui \
+      -only=vsphere-iso.upgrade \
+      -var vm_name=$DRONE_BUILD_NUMBER-${DRONE_COMMIT_SHA:0:10} \
+      -var docker_username=$${DOCKER_USERNAME} \
+      -var docker_password=$${DOCKER_PASSWORD} \
+      -var repo_username=$${REPO_USERNAME} \
+      -var repo_password=$${REPO_PASSWORD} \
+      -var ssh_password=$${SSH_PASSWORD} \
+      -var vsphere_password=$${VSPHERE_PASSWORD} \
+      -var k8s_version=$K8S_VERSION \
+      ./packer
+  environment:
+    DOCKER_USERNAME:
+      from_secret: docker_username
+    DOCKER_PASSWORD:
+      from_secret: docker_password
+    # PACKER_LOG: 1
+    REPO_USERNAME:
+      from_secret: repo_username
+    REPO_PASSWORD:
+      from_secret: repo_password
+    SSH_PASSWORD:
+      from_secret: ssh_password
+    VSPHERE_PASSWORD:
+      from_secret: vsphere_password
+  volumes:
+  - name: output
+    path: /output
+  - name: scratch
+    path: /scratch
+
 - name: Remove temporary resources
+  depends_on:
+  - Kubernetes Bootstrap Appliance
+  - Kubernetes Upgrade Appliance
   image: bv11-cr01.bessems.eu/library/packer-extended
   commands:
   - |

From 6942c33ae8d6eb98ec13620e50a327e19fd7059c Mon Sep 17 00:00:00 2001
From: Danny Bessems <danny@bessems.eu>
Date: Mon, 3 Apr 2023 16:26:38 +0200
Subject: [PATCH 21/51] Fix Ansible templating;DRY

---
 .drone.yml                                    | 43 +++++++++++--------
 .../roles/workloadcluster/tasks/gitops.yml    |  6 +--
 packer/build.pkr.hcl                          |  2 +
 3 files changed, 30 insertions(+), 21 deletions(-)

diff --git a/.drone.yml b/.drone.yml
index 4877337..f56f4aa 100644
--- a/.drone.yml
+++ b/.drone.yml
@@ -13,15 +13,38 @@ volumes:
 steps:
 - name: Debugging information
   image: bv11-cr01.bessems.eu/library/packer-extended
+  pull: always
   commands:
   - ansible --version
   - ovftool --version
   - packer --version
   - yamllint --version
 
+- name: Linting
+  image: bv11-cr01.bessems.eu/library/packer-extended
+  pull: always
+  commands:
+  - |
+    yamllint -d "{extends: relaxed, rules: {line-length: disable}}" \
+      ansible \
+      packer/preseed/UbuntuServer22.04/user-data \
+      scripts
+
+- name: Install Ansible Galaxy collections
+  image: bv11-cr01.bessems.eu/library/packer-extended
+  pull: always
+  commands:
+    - |
+    ansible-galaxy install \
+      -r ansible/requirements.yml \
+      -p /scratch/ansible-galaxy
+  volumes:
+  - name: scratch
+    path: /scratch
+
 - name: Kubernetes Bootstrap Appliance
   depends_on:
-  - Debugging information
+  - Install Ansible Galaxy collections
   image: bv11-cr01.bessems.eu/library/packer-extended
   pull: always
   commands:
@@ -30,14 +53,6 @@ steps:
       packer/preseed/UbuntuServer22.04/user-data
   - |
     export K8S_VERSION=$(yq '.components.clusterapi.workload.version.k8s' < ./ansible/vars/metacluster.yml)
-  - |
-    yamllint -d "{extends: relaxed, rules: {line-length: disable}}" \
-      ansible \
-      packer/preseed/UbuntuServer22.04/user-data \
-      scripts
-  - |
-    ansible-galaxy install \
-      -r ansible/requirements.yml
   - |
     packer init -upgrade \
       ./packer
@@ -87,7 +102,7 @@ steps:
 
 - name: Kubernetes Upgrade Appliance
   depends_on:
-  - Debugging information
+  - Install Ansible Galaxy collections
   image: bv11-cr01.bessems.eu/library/packer-extended
   pull: always
   commands:
@@ -96,14 +111,6 @@ steps:
       packer/preseed/UbuntuServer22.04/user-data
   - |
     export K8S_VERSION=$(yq '.components.clusterapi.workload.version.k8s' < ./ansible/vars/metacluster.yml)
-  - |
-    yamllint -d "{extends: relaxed, rules: {line-length: disable}}" \
-      ansible \
-      packer/preseed/UbuntuServer22.04/user-data \
-      scripts
-  - |
-    ansible-galaxy install \
-      -r ansible/requirements.yml
   - |
     packer init -upgrade \
       ./packer
diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/gitops.yml b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/gitops.yml
index 1f32468..6fffb60 100644
--- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/gitops.yml
+++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/gitops.yml
@@ -59,9 +59,9 @@
             secret: argocd-cluster-{{ vapp['workloadcluster.name'] | lower }}
             url: https://{{ vapp['workloadcluster.vip'] }}:6443
           kubeconfig:
-            ca: (secret_workloadcluster_kubeconfig.resources[0].data.value | b64decode | from_yaml).clusters[0].cluster['certificate-authority-data'] | b64encode
-            certificate: (secret_workloadcluster_kubeconfig.resources[0].data.value | b64decode | from_yaml).users[0].user['client-certificate-data'] | b64encode
-            key: (secret_workloadcluster_kubeconfig.resources[0].data.value | b64decode | from_yaml).users[0].user['client-key-data'] | b64encode
+            ca: "{{ (secret_workloadcluster_kubeconfig.resources[0].data.value | b64decode | from_yaml).clusters[0].cluster['certificate-authority-data'] | b64encode }}"
+            certificate: "{{ (secret_workloadcluster_kubeconfig.resources[0].data.value | b64decode | from_yaml).users[0].user['client-certificate-data'] | b64encode }}"
+            key: "{{ (secret_workloadcluster_kubeconfig.resources[0].data.value | b64decode | from_yaml).users[0].user['client-key-data'] | b64encode }}"
 
     - name: Configure workload-cluster GitOps repository
       ansible.builtin.template:
diff --git a/packer/build.pkr.hcl b/packer/build.pkr.hcl
index fa6e9d9..463ae82 100644
--- a/packer/build.pkr.hcl
+++ b/packer/build.pkr.hcl
@@ -34,6 +34,8 @@ build {
       "PYTHONUNBUFFERED=1"
     ]
     use_proxy        = "false"
+    collections-path = "/scratch/ansible-galaxy"
+
     extra_arguments  = [
       "--extra-vars", "appliancetype=${source.name}",
       "--extra-vars", "ansible_ssh_pass=${var.ssh_password}",

From cde92b4514529bdad0fcd30c99c096c560e5a88a Mon Sep 17 00:00:00 2001
From: Danny Bessems <danny@bessems.eu>
Date: Mon, 3 Apr 2023 16:29:16 +0200
Subject: [PATCH 22/51] Fix indentation

---
 .drone.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.drone.yml b/.drone.yml
index f56f4aa..4e31ec7 100644
--- a/.drone.yml
+++ b/.drone.yml
@@ -34,7 +34,7 @@ steps:
   image: bv11-cr01.bessems.eu/library/packer-extended
   pull: always
   commands:
-    - |
+  - |
     ansible-galaxy install \
       -r ansible/requirements.yml \
       -p /scratch/ansible-galaxy

From 5a82c9e122e72ea6bfbcbd744711c6c1a274f5fc Mon Sep 17 00:00:00 2001
From: Danny Bessems <danny@bessems.eu>
Date: Mon, 3 Apr 2023 16:32:05 +0200
Subject: [PATCH 23/51] Fix key name;Fix task dependencies

---
 .drone.yml           | 4 ++++
 packer/build.pkr.hcl | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/.drone.yml b/.drone.yml
index 4e31ec7..b8bc581 100644
--- a/.drone.yml
+++ b/.drone.yml
@@ -21,6 +21,8 @@ steps:
   - yamllint --version
 
 - name: Linting
+  depends_on:
+  - Debugging information
   image: bv11-cr01.bessems.eu/library/packer-extended
   pull: always
   commands:
@@ -31,6 +33,8 @@ steps:
       scripts
 
 - name: Install Ansible Galaxy collections
+  depends_on:
+  - Linting
   image: bv11-cr01.bessems.eu/library/packer-extended
   pull: always
   commands:
diff --git a/packer/build.pkr.hcl b/packer/build.pkr.hcl
index 463ae82..af6b8c1 100644
--- a/packer/build.pkr.hcl
+++ b/packer/build.pkr.hcl
@@ -34,7 +34,7 @@ build {
       "PYTHONUNBUFFERED=1"
     ]
     use_proxy        = "false"
-    collections-path = "/scratch/ansible-galaxy"
+    collections_path = "/scratch/ansible-galaxy"
 
     extra_arguments  = [
       "--extra-vars", "appliancetype=${source.name}",

From 8e766177940cfebeb877d4fb5b23c8bed020ca59 Mon Sep 17 00:00:00 2001
From: Danny Bessems <danny@bessems.eu>
Date: Mon, 3 Apr 2023 17:25:45 +0200
Subject: [PATCH 24/51] Fix repository path;Add chart values;Fix ansible galaxy
 cli syntax

---
 .drone.yml                                    |  2 +-
 ansible/roles/assets/tasks/main.yml           |  3 +-
 .../roles/workloadcluster/tasks/gitops.yml    |  2 +-
 .../bootstrap/templates/applicationset.j2     |  2 +-
 ansible/vars/workloadcluster.yml              | 32 +++++++++++++++++++
 5 files changed, 37 insertions(+), 4 deletions(-)

diff --git a/.drone.yml b/.drone.yml
index b8bc581..5073570 100644
--- a/.drone.yml
+++ b/.drone.yml
@@ -39,7 +39,7 @@ steps:
   pull: always
   commands:
   - |
-    ansible-galaxy install \
+    ansible-galaxy collection install \
       -r ansible/requirements.yml \
       -p /scratch/ansible-galaxy
   volumes:
diff --git a/ansible/roles/assets/tasks/main.yml b/ansible/roles/assets/tasks/main.yml
index 23ba858..ad8bc81 100644
--- a/ansible/roles/assets/tasks/main.yml
+++ b/ansible/roles/assets/tasks/main.yml
@@ -16,7 +16,8 @@
     - /opt/metacluster/helm-charts
     - /opt/metacluster/k3s
     - /opt/metacluster/kube-vip
-    - /opt/workloadcluster/git-repositories/gitops
+    - /opt/workloadcluster/git-repositories/gitops/charts
+    - /opt/workloadcluster/git-repositories/gitops/values
     - /opt/workloadcluster/helm-charts
     - /opt/workloadcluster/node-templates
     - /var/lib/rancher/k3s/agent/images
diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/gitops.yml b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/gitops.yml
index 6fffb60..8b28efc 100644
--- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/gitops.yml
+++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/gitops.yml
@@ -8,7 +8,7 @@
 - name: Create hard-links to populate new git-repository
   ansible.builtin.shell:
     cmd: >-
-      cp -lr {{ item.path }}/ /opt/workloadcluster/git-repositories/gitops/
+      cp -lr {{ item.path }}/ /opt/workloadcluster/git-repositories/gitops/charts
   loop: "{{ helm_charts.files }}"
   loop_control:
     label: "{{ item.path | basename }}"
diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/applicationset.j2 b/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/applicationset.j2
index 4160fe4..c2ae97d 100644
--- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/applicationset.j2
+++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/applicationset.j2
@@ -9,7 +9,7 @@ spec:
       repoURL: {{ _template.repository.url }}
       revision: HEAD
       directories:
-      - path: applicationset/*
+      - path: charts/*
   template:
     metadata:
       name: {% raw %}'{{ path.basename }}'{% endraw +%}
diff --git a/ansible/vars/workloadcluster.yml b/ansible/vars/workloadcluster.yml
index 230fcce..3e090ac 100644
--- a/ansible/vars/workloadcluster.yml
+++ b/ansible/vars/workloadcluster.yml
@@ -15,6 +15,38 @@ downstream:
       chart_values: !unsafe |
         defaultSettings:
           defaultDataPath: /mnt/blockstorage
+        global:
+          cattle:
+            systemDefaultRegistry: registry.{{ vapp['metacluster.fqdn'] }}
+        image:
+          longhorn:
+            engine:
+              repository: library/docker.io/longhornio/longhorn-engine
+            manager:
+              repository: library/docker.io/longhornio/longhorn-manager
+            ui:
+              repository: library/docker.io/longhornio/longhorn-ui
+            instanceManager:
+              repository: library/docker.io/longhornio/longhorn-instance-manager
+            shareManager:
+              repository: library/docker.io/longhornio/longhorn-share-manager
+            backingImageManager:
+              repository: library/docker.io/longhornio/backing-image-manager
+            supportBundleKit:
+              repository: library/docker.io/longhornio/support-bundle-kit
+          csi:
+            attacher:
+              repository: library/docker.io/longhornio/csi-attacher
+            provisioner:
+              repository: library/docker.io/longhornio/csi-provisioner
+            nodeDriverRegistrar:
+              repository: library/docker.io/longhornio/csi-node-driver-registrar
+            resizer:
+              repository: library/docker.io/longhornio/csi-resizer
+            snapshotter:
+              repository: library/docker.io/longhornio/csi-snapshotter
+            livenessProbe:
+              repository: library/docker.io/longhornio/livenessprobe
 
     sealed-secrets:
       version: 2.8.1  # (= Sealed Secrets v0.20.2)

From 9ff0e09625e939aa2294e7b016b99478c8e539ef Mon Sep 17 00:00:00 2001
From: Danny Bessems <danny@bessems.eu>
Date: Mon, 3 Apr 2023 21:20:10 +0200
Subject: [PATCH 25/51] Test ansible collection paths

---
 .drone.yml           | 2 +-
 packer/build.pkr.hcl | 1 -
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/.drone.yml b/.drone.yml
index 5073570..7d68124 100644
--- a/.drone.yml
+++ b/.drone.yml
@@ -41,7 +41,7 @@ steps:
   - |
     ansible-galaxy collection install \
       -r ansible/requirements.yml \
-      -p /scratch/ansible-galaxy
+      -p ./ansible
   volumes:
   - name: scratch
     path: /scratch
diff --git a/packer/build.pkr.hcl b/packer/build.pkr.hcl
index af6b8c1..1518eee 100644
--- a/packer/build.pkr.hcl
+++ b/packer/build.pkr.hcl
@@ -34,7 +34,6 @@ build {
       "PYTHONUNBUFFERED=1"
     ]
     use_proxy        = "false"
-    collections_path = "/scratch/ansible-galaxy"
 
     extra_arguments  = [
       "--extra-vars", "appliancetype=${source.name}",

From d0c4251e06b06b088613ea8eeeb072e8ec9f3754 Mon Sep 17 00:00:00 2001
From: Danny Bessems <danny@bessems.eu>
Date: Mon, 3 Apr 2023 21:37:09 +0200
Subject: [PATCH 26/51] Configure registry mirrors on workload-cluster
 nodes;Test ansible collection paths #2

---
 .drone.yml                                    |  2 +-
 .../workloadcluster/tasks/clusterapi.yml      |  4 +++
 .../kustomization.cluster-template.j2         | 36 +++++++++++++++++++
 packer/build.pkr.hcl                          |  1 +
 4 files changed, 42 insertions(+), 1 deletion(-)

diff --git a/.drone.yml b/.drone.yml
index 7d68124..f5c154c 100644
--- a/.drone.yml
+++ b/.drone.yml
@@ -41,7 +41,7 @@ steps:
   - |
     ansible-galaxy collection install \
       -r ansible/requirements.yml \
-      -p ./ansible
+      -p ./ansible/collections
   volumes:
   - name: scratch
     path: /scratch
diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/clusterapi.yml b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/clusterapi.yml
index 68f92cb..6d0dbff 100644
--- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/clusterapi.yml
+++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/clusterapi.yml
@@ -82,6 +82,10 @@
       rootca: "{{ stepca_cm_certs.resources[0].data['root_ca.crt'] }}"
       runcmds:
         - update-ca-certificates
+      registries:
+        # This should obviously be a dynamic list, but testing first!
+        - docker.io
+        - gcr.io
 
 - name: Store custom cluster-template
   ansible.builtin.copy:
diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/kustomization.cluster-template.j2 b/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/kustomization.cluster-template.j2
index 03b8206..e116fd7 100644
--- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/kustomization.cluster-template.j2
+++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/kustomization.cluster-template.j2
@@ -47,6 +47,21 @@ patchesStrategicMerge:
       template:
         spec:
           files:
+          - content: |
+                [plugins."io.containerd.grpc.v1.cri".registry]
+                  config_path = "/etc/containerd/certs.d"
+            append: true
+            path: /etc/containerd/config.toml
+{% for registry in _template.registries %}
+          - content: |
+              server = "https://{{ registry }}"
+
+              [host."https://registry.{{ _template.network.fqdn }}/v2/library/{{ registry }}"]
+                capabilities = ["pull", "resolve"]
+                override_path = true
+            owner: root:root
+            path: /etc/containerd/certs.d/{{ registry }}/hosts.toml
+{% endfor %}
           - content: |
               network: {config: disabled}
             owner: root:root
@@ -103,6 +118,27 @@ patchesJson6902:
       kind: KubeadmControlPlane
       name: .*
     patch: |-
+      - op: add
+        path: /spec/kubeadmConfigSpec/files/-
+        value:
+          content: |
+              [plugins."io.containerd.grpc.v1.cri".registry]
+                config_path = "/etc/containerd/certs.d"
+          append: true
+          path: /etc/containerd/config.toml
+{% for registry in _template.registries %}
+      - op: add
+        path: /spec/kubeadmConfigSpec/files/-
+        value:
+          content: |
+            server = "https://{{ registry }}"
+
+            [host."https://registry.{{ _template.network.fqdn }}/v2/library/{{ registry }}"]
+              capabilities = ["pull", "resolve"]
+              override_path = true
+          owner: root:root
+          path: /etc/containerd/certs.d/{{ registry }}/hosts.toml
+{% endfor %}
       - op: add
         path: /spec/kubeadmConfigSpec/files/-
         value:
diff --git a/packer/build.pkr.hcl b/packer/build.pkr.hcl
index 1518eee..2c2e4ce 100644
--- a/packer/build.pkr.hcl
+++ b/packer/build.pkr.hcl
@@ -34,6 +34,7 @@ build {
       "PYTHONUNBUFFERED=1"
     ]
     use_proxy        = "false"
+    collections_path = "ansible/collections"
 
     extra_arguments  = [
       "--extra-vars", "appliancetype=${source.name}",

From 915660f618186a85676719d249ce057ca3fea207 Mon Sep 17 00:00:00 2001
From: Danny Bessems <danny@bessems.eu>
Date: Tue, 4 Apr 2023 10:47:28 +0200
Subject: [PATCH 27/51] Housekeeping;Populate all registry mirrors;Disable
 manifest image reference workaround;Add missing key;Remove redundant filter

---
 .../roles/metacluster/tasks/init.yml          |  2 +-
 .../workloadcluster/tasks/clusterapi.yml      | 44 +++++++------------
 .../roles/workloadcluster/tasks/gitops.yml    |  7 +--
 .../common/templates/registries.j2            |  6 +--
 4 files changed, 25 insertions(+), 34 deletions(-)

diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/init.yml b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/init.yml
index f01c1fe..ddbcd7e 100644
--- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/init.yml
+++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/init.yml
@@ -78,6 +78,6 @@
     src: registries.j2
   vars:
     _template:
-      data: "{{ source_registries }}"
+      registries: "{{ source_registries }}"
       hv:
         fqdn: "{{ vapp['metacluster.fqdn'] }}"
diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/clusterapi.yml b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/clusterapi.yml
index 6d0dbff..b5822a5 100644
--- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/clusterapi.yml
+++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/clusterapi.yml
@@ -52,22 +52,22 @@
         version: "{{ components.clusterapi.workload.version.k8s }}"
         vip: "{{ vapp['workloadcluster.vip'] }}"
 
-- name: WORKAROUND - Update image references to use local registry
-  ansible.builtin.replace:
-    dest: "{{ item }}"
-    regexp: '([ ]+image:[ "]+)(?!({{ _template.pattern }}|"{{ _template.pattern }}))'
-    replace: '\1{{ _template.pattern }}'
-  vars:
-    fileglobs:
-      - "{{ query('ansible.builtin.fileglob', '/opt/metacluster/cluster-api/cni-calico/' ~ components.clusterapi.workload.version.calico ~ '/*.yaml') }}"
-      - "{{ query('ansible.builtin.fileglob', '/opt/metacluster/cluster-api/infrastructure-vsphere/' ~ components.clusterapi.management.version.infrastructure_vsphere ~ '/*.yaml') }}"
-    _template:
-      pattern: registry.{{ vapp['metacluster.fqdn'] }}/library/
-  loop: "{{ fileglobs[0:] | flatten | select }}"
-  loop_control:
-    label: "{{ item | basename }}"
-  when:
-    - item is not search("components.yaml|metadata.yaml")
+# - name: WORKAROUND - Update image references to use local registry
+#   ansible.builtin.replace:
+#     dest: "{{ item }}"
+#     regexp: '([ ]+image:[ "]+)(?!({{ _template.pattern }}|"{{ _template.pattern }}))'
+#     replace: '\1{{ _template.pattern }}'
+#   vars:
+#     fileglobs:
+#       - "{{ query('ansible.builtin.fileglob', '/opt/metacluster/cluster-api/cni-calico/' ~ components.clusterapi.workload.version.calico ~ '/*.yaml') }}"
+#       - "{{ query('ansible.builtin.fileglob', '/opt/metacluster/cluster-api/infrastructure-vsphere/' ~ components.clusterapi.management.version.infrastructure_vsphere ~ '/*.yaml') }}"
+#     _template:
+#       pattern: registry.{{ vapp['metacluster.fqdn'] }}/library/
+#   loop: "{{ fileglobs[0:] | flatten | select }}"
+#   loop_control:
+#     label: "{{ item | basename }}"
+#   when:
+#     - item is not search("components.yaml|metadata.yaml")
 
 - name: Generate kustomization template
   ansible.builtin.template:
@@ -82,10 +82,7 @@
       rootca: "{{ stepca_cm_certs.resources[0].data['root_ca.crt'] }}"
       runcmds:
         - update-ca-certificates
-      registries:
-        # This should obviously be a dynamic list, but testing first!
-        - docker.io
-        - gcr.io
+      registries: "{{ source_registries }}"
 
 - name: Store custom cluster-template
   ansible.builtin.copy:
@@ -246,13 +243,6 @@
 - name: Trigger handlers
   ansible.builtin.meta: flush_handlers
 
-# - name: Apply workload cluster manifest
-#   kubernetes.core.k8s:
-#     definition: >-
-#       {{ clusterctl_newcluster.stdout }}
-#     wait: true
-#     kubeconfig: "{{ kubeconfig.path }}"
-
 - name: Wait for cluster to be available
   ansible.builtin.shell:
     cmd: >-
diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/gitops.yml b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/gitops.yml
index 8b28efc..8834451 100644
--- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/gitops.yml
+++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/gitops.yml
@@ -45,6 +45,7 @@
         kind: Secret
         name: "{{ vapp['workloadcluster.name'] }}-kubeconfig"
         namespace: default
+        kubeconfig: "{{ kubeconfig.path }}"
       register: secret_workloadcluster_kubeconfig
 
     - name: Register workload-cluster in argo-cd
@@ -59,9 +60,9 @@
             secret: argocd-cluster-{{ vapp['workloadcluster.name'] | lower }}
             url: https://{{ vapp['workloadcluster.vip'] }}:6443
           kubeconfig:
-            ca: "{{ (secret_workloadcluster_kubeconfig.resources[0].data.value | b64decode | from_yaml).clusters[0].cluster['certificate-authority-data'] | b64encode }}"
-            certificate: "{{ (secret_workloadcluster_kubeconfig.resources[0].data.value | b64decode | from_yaml).users[0].user['client-certificate-data'] | b64encode }}"
-            key: "{{ (secret_workloadcluster_kubeconfig.resources[0].data.value | b64decode | from_yaml).users[0].user['client-key-data'] | b64encode }}"
+            ca: "{{ (secret_workloadcluster_kubeconfig.resources[0].data.value | b64decode | from_yaml).clusters[0].cluster['certificate-authority-data'] }}"
+            certificate: "{{ (secret_workloadcluster_kubeconfig.resources[0].data.value | b64decode | from_yaml).users[0].user['client-certificate-data'] }}"
+            key: "{{ (secret_workloadcluster_kubeconfig.resources[0].data.value | b64decode | from_yaml).users[0].user['client-key-data'] }}"
 
     - name: Configure workload-cluster GitOps repository
       ansible.builtin.template:
diff --git a/ansible/roles/firstboot/files/ansible_payload/common/templates/registries.j2 b/ansible/roles/firstboot/files/ansible_payload/common/templates/registries.j2
index 45cde78..8345db3 100644
--- a/ansible/roles/firstboot/files/ansible_payload/common/templates/registries.j2
+++ b/ansible/roles/firstboot/files/ansible_payload/common/templates/registries.j2
@@ -1,8 +1,8 @@
 mirrors:
-{% for entry in _template.data %}
-  {{ entry }}:
+{% for registry in _template.registries %}
+  {{ registry }}:
     endpoint:
       - https://registry.{{ _template.hv.fqdn }}
     rewrite:
-      "(.*)": "library/{{ entry }}/$1"
+      "(.*)": "library/{{ registry }}/$1"
 {% endfor %}

From 56a33134a0fa14df6b55c263331fbe3b791a9733 Mon Sep 17 00:00:00 2001
From: Danny Bessems <danny@bessems.eu>
Date: Tue, 4 Apr 2023 17:22:39 +0200
Subject: [PATCH 28/51] Housekeeping;Move inclusterippool to gitops;Delete
 temporary manifests;Align resource naming;Remove redundant config;Add helm
 configuration

---
 .../roles/assets/tasks/containerimages.yml    |   2 +-
 ansible/roles/assets/tasks/helm.yml           |   2 +-
 .../roles/metacluster/tasks/gitops.yml        |   2 +-
 .../workloadcluster/tasks/clusterapi.yml      |  64 +++-----
 .../roles/workloadcluster/tasks/gitops.yml    | 139 +++++++-----------
 .../bootstrap/templates/applicationset.j2     |  17 ++-
 .../templates/serviceaccount.j2.DISABLED      |  27 ----
 ansible/vars/workloadcluster.yml              |  34 +----
 8 files changed, 94 insertions(+), 193 deletions(-)
 delete mode 100644 ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/serviceaccount.j2.DISABLED

diff --git a/ansible/roles/assets/tasks/containerimages.yml b/ansible/roles/assets/tasks/containerimages.yml
index 504f49f..d09f7de 100644
--- a/ansible/roles/assets/tasks/containerimages.yml
+++ b/ansible/roles/assets/tasks/containerimages.yml
@@ -27,7 +27,7 @@
 - name: Parse workloadcluster helm charts for container images
   ansible.builtin.shell:
     cmd: "{{ item.value.parse_logic }}"
-    chdir: /opt/workloadcluster/helm-charts/{{ item.key }}
+    chdir: /opt/workloadcluster/helm-charts/{{ item.value.namespace }}/{{ item.key }}
   register: chartimages_workloadcluster
   loop: "{{ lookup('ansible.builtin.dict', downstream.helm_charts) }}"
   loop_control:
diff --git a/ansible/roles/assets/tasks/helm.yml b/ansible/roles/assets/tasks/helm.yml
index e511453..03071c6 100644
--- a/ansible/roles/assets/tasks/helm.yml
+++ b/ansible/roles/assets/tasks/helm.yml
@@ -20,7 +20,7 @@
 
 - name: Fetch helm charts for workloadcluster
   ansible.builtin.command:
-    cmd: helm fetch {{ item.value.chart }} --untar --version {{ item.value.version }}
+    cmd: helm fetch {{ item.value.chart }} --untardir ./{{ item.value.namespace }} --untar --version {{ item.value.version }}
     chdir: /opt/workloadcluster/helm-charts
   register: helmcharts_workloadcluster
   loop: "{{ lookup('ansible.builtin.dict', downstream.helm_charts) }}"
diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/gitops.yml b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/gitops.yml
index 4d00ff4..c2799b5 100644
--- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/gitops.yml
+++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/gitops.yml
@@ -55,7 +55,7 @@
       vars:
         _template:
           application:
-            name: argocd-applicationset-metacluster
+            name: applicationset-metacluster
             namespace: argo-cd
           cluster:
             url: https://kubernetes.default.svc
diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/clusterapi.yml b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/clusterapi.yml
index b5822a5..360e138 100644
--- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/clusterapi.yml
+++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/clusterapi.yml
@@ -52,23 +52,6 @@
         version: "{{ components.clusterapi.workload.version.k8s }}"
         vip: "{{ vapp['workloadcluster.vip'] }}"
 
-# - name: WORKAROUND - Update image references to use local registry
-#   ansible.builtin.replace:
-#     dest: "{{ item }}"
-#     regexp: '([ ]+image:[ "]+)(?!({{ _template.pattern }}|"{{ _template.pattern }}))'
-#     replace: '\1{{ _template.pattern }}'
-#   vars:
-#     fileglobs:
-#       - "{{ query('ansible.builtin.fileglob', '/opt/metacluster/cluster-api/cni-calico/' ~ components.clusterapi.workload.version.calico ~ '/*.yaml') }}"
-#       - "{{ query('ansible.builtin.fileglob', '/opt/metacluster/cluster-api/infrastructure-vsphere/' ~ components.clusterapi.management.version.infrastructure_vsphere ~ '/*.yaml') }}"
-#     _template:
-#       pattern: registry.{{ vapp['metacluster.fqdn'] }}/library/
-#   loop: "{{ fileglobs[0:] | flatten | select }}"
-#   loop_control:
-#     label: "{{ item | basename }}"
-#   when:
-#     - item is not search("components.yaml|metadata.yaml")
-
 - name: Generate kustomization template
   ansible.builtin.template:
     src: kustomization.cluster-template.j2
@@ -155,6 +138,21 @@
         -f {{ capi_clustermanifest.path }}/new-cluster.yaml \
         -o {{ capi_clustermanifest.path }}/manifests
 
+- name: Create in-cluster IpPool
+  ansible.builtin.template:
+    src: ippool.j2
+    dest: "{{ capi_clustermanifest.path }}/manifests/inclusterippool-{{ _template.cluster.name }}.yml"
+  vars:
+    _template:
+      cluster:
+        name: "{{ vapp['workloadcluster.name'] | lower }}"
+        namespace: default
+        network:
+          startip: "{{ vapp['ippool.startip'] }}"
+          endip: "{{ vapp['ippool.endip'] }}"
+          prefix: "{{ vapp['guestinfo.prefixlength'] }}"
+          gateway: "{{ vapp['guestinfo.gateway'] }}"
+
 - name: Initialize/Push git repository
   ansible.builtin.shell:
     cmd: |
@@ -168,6 +166,12 @@
       git push https://administrator:{{ vapp['metacluster.password'] | urlencode }}@git.{{ vapp['metacluster.fqdn'] }}/mc/GitOps.ClusterAPI.git --all
     chdir: "{{ capi_clustermanifest.path }}"
 
+- name: Cleanup tempfolder
+  ansible.builtin.file:
+    path: "{{ capi_clustermanifest.path }}"
+    state: absent
+  when: capi_clustermanifest.path is defined
+
 - name: Configure Cluster API repository
   ansible.builtin.template:
     src: gitrepo.j2
@@ -183,30 +187,6 @@
   notify:
     - Apply manifests
 
-# Temporarily disabled until manifests are properly managed by gitops
-# - name: Cleanup tempfolder
-#   ansible.builtin.file:
-#     path: "{{ capi_clustermanifest.path }}"
-#     state: absent
-#   when: capi_clustermanifest.path is defined
-
-# TODO: Move to gitops
-- name: Create in-cluster IpPool
-  kubernetes.core.k8s:
-    template: ippool.j2
-    state: present
-    kubeconfig: "{{ kubeconfig.path }}"
-  vars:
-    _template:
-      cluster:
-        name: "{{ vapp['workloadcluster.name'] | lower }}"
-        namespace: default
-        network:
-          startip: "{{ vapp['ippool.startip'] }}"
-          endip: "{{ vapp['ippool.endip'] }}"
-          prefix: "{{ vapp['guestinfo.prefixlength'] }}"
-          gateway: "{{ vapp['guestinfo.gateway'] }}"
-
 - name: WORKAROUND - Wait for ingress ACME requests to complete
   ansible.builtin.shell:
     cmd: >-
@@ -228,7 +208,7 @@
   vars:
     _template:
       application:
-        name: argocd-application-clusterapi
+        name: application-clusterapi-workloadcluster
         namespace: argo-cd
       cluster:
         name: https://kubernetes.default.svc
diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/gitops.yml b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/gitops.yml
index 8834451..72845ea 100644
--- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/gitops.yml
+++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/gitops.yml
@@ -26,89 +26,64 @@
       git push https://administrator:{{ vapp['metacluster.password'] | urlencode }}@git.{{ vapp['metacluster.fqdn'] }}/wl/GitOps.Config.git --all
     chdir: /opt/workloadcluster/git-repositories/gitops
 
-- block:
+- name: Retrieve workload-cluster kubeconfig
+  kubernetes.core.k8s_info:
+    kind: Secret
+    name: "{{ vapp['workloadcluster.name'] }}-kubeconfig"
+    namespace: default
+    kubeconfig: "{{ kubeconfig.path }}"
+  register: secret_workloadcluster_kubeconfig
 
-    # - name: Generate service account in workload-cluster
-    #   kubernetes.core.k8s:
-    #     template: serviceaccount.j2
-    #     state: present
+- name: Register workload-cluster in argo-cd
+  kubernetes.core.k8s:
+    template: cluster.j2
+    state: present
+    kubeconfig: "{{ kubeconfig.path }}"
+  vars:
+    _template:
+      cluster:
+        name: "{{ vapp['workloadcluster.name'] | lower }}"
+        secret: argocd-cluster-{{ vapp['workloadcluster.name'] | lower }}
+        url: https://{{ vapp['workloadcluster.vip'] }}:6443
+      kubeconfig:
+        ca: "{{ (secret_workloadcluster_kubeconfig.resources[0].data.value | b64decode | from_yaml).clusters[0].cluster['certificate-authority-data'] }}"
+        certificate: "{{ (secret_workloadcluster_kubeconfig.resources[0].data.value | b64decode | from_yaml).users[0].user['client-certificate-data'] }}"
+        key: "{{ (secret_workloadcluster_kubeconfig.resources[0].data.value | b64decode | from_yaml).users[0].user['client-key-data'] }}"
 
-    # - name: Retrieve service account bearer token
-    #   kubernetes.core.k8s_info:
-    #     kind: Secret
-    #     name: "{{ _template.account.name }}-secret"
-    #     namespace: "{{ _template.account.namespace }}"
-    #   register: workloadcluster_bearertoken
+- name: Configure workload-cluster GitOps repository
+  ansible.builtin.template:
+    src: gitrepo.j2
+    dest: /var/lib/rancher/k3s/server/manifests/{{ _template.name }}-manifest.yaml
+    owner: root
+    group: root
+    mode: 0600
+  vars:
+    _template:
+      name: argocd-gitrepo-wl-gitopsconfig
+      namespace: argo-cd
+      url: https://git.{{ vapp['metacluster.fqdn'] }}/wl/GitOps.Config.git
+  notify:
+    - Apply manifests
 
-    - name: Retrieve workload-cluster kubeconfig
-      kubernetes.core.k8s_info:
-        kind: Secret
-        name: "{{ vapp['workloadcluster.name'] }}-kubeconfig"
-        namespace: default
-        kubeconfig: "{{ kubeconfig.path }}"
-      register: secret_workloadcluster_kubeconfig
+- name: Create applicationset
+  ansible.builtin.template:
+    src: applicationset.j2
+    dest: /var/lib/rancher/k3s/server/manifests/{{ _template.application.name }}-manifest.yaml
+    owner: root
+    group: root
+    mode: 0600
+  vars:
+    _template:
+      application:
+        name: applicationset-workloadcluster
+        namespace: argo-cd
+      cluster:
+        url: https://{{ vapp['workloadcluster.vip'] }}:6443
+      repository:
+        url: https://git.{{ vapp['metacluster.fqdn'] }}/wl/GitOps.Config.git
+        revision: main
+  notify:
+    - Apply manifests
 
-    - name: Register workload-cluster in argo-cd
-      kubernetes.core.k8s:
-        template: cluster.j2
-        state: present
-        kubeconfig: "{{ kubeconfig.path }}"
-      vars:
-        _template:
-          cluster:
-            name: "{{ vapp['workloadcluster.name'] | lower }}"
-            secret: argocd-cluster-{{ vapp['workloadcluster.name'] | lower }}
-            url: https://{{ vapp['workloadcluster.vip'] }}:6443
-          kubeconfig:
-            ca: "{{ (secret_workloadcluster_kubeconfig.resources[0].data.value | b64decode | from_yaml).clusters[0].cluster['certificate-authority-data'] }}"
-            certificate: "{{ (secret_workloadcluster_kubeconfig.resources[0].data.value | b64decode | from_yaml).users[0].user['client-certificate-data'] }}"
-            key: "{{ (secret_workloadcluster_kubeconfig.resources[0].data.value | b64decode | from_yaml).users[0].user['client-key-data'] }}"
-
-    - name: Configure workload-cluster GitOps repository
-      ansible.builtin.template:
-        src: gitrepo.j2
-        dest: /var/lib/rancher/k3s/server/manifests/{{ _template.name }}-manifest.yaml
-        owner: root
-        group: root
-        mode: 0600
-      vars:
-        _template:
-          name: argocd-gitrepo-wl-gitopsconfig
-          namespace: argo-cd
-          url: https://git.{{ vapp['metacluster.fqdn'] }}/wl/GitOps.Config.git
-      notify:
-        - Apply manifests
-
-    - name: Create applicationset
-      ansible.builtin.template:
-        src: applicationset.j2
-        dest: /var/lib/rancher/k3s/server/manifests/{{ _template.application.name }}-manifest.yaml
-        owner: root
-        group: root
-        mode: 0600
-      vars:
-        _template:
-          application:
-            name: argocd-applicationset-workloadcluster
-            namespace: argo-cd
-          cluster:
-            url: https://{{ vapp['workloadcluster.vip'] }}:6443
-          repository:
-            url: https://git.{{ vapp['metacluster.fqdn'] }}/wl/GitOps.Config.git
-
-      notify:
-        - Apply manifests
-
-    - name: Trigger handlers
-      ansible.builtin.meta: flush_handlers
-
-  # vars:
-  #   _template:
-  #     account:
-  #       name: argocd-sa
-  #       namespace: default
-  #     clusterrolebinding:
-  #       name: argocd-crb
-  module_defaults:
-    group/k8s:
-      kubeconfig: "{{ capi_kubeconfig.path }}"
+- name: Trigger handlers
+  ansible.builtin.meta: flush_handlers
diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/applicationset.j2 b/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/applicationset.j2
index c2ae97d..cdfd461 100644
--- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/applicationset.j2
+++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/applicationset.j2
@@ -7,22 +7,25 @@ spec:
   generators:
   - git:
       repoURL: {{ _template.repository.url }}
-      revision: HEAD
+      revision: {{ _template.repository.revision }}
       directories:
-      - path: charts/*
+      - path: charts/*/*
   template:
     metadata:
-      name: {% raw %}'{{ path.basename }}'{% endraw +%}
+      name: application-{% raw %}{{ path.basename }}{% endraw +%}
     spec:
       project: default
       syncPolicy:
         automated:
           prune: true
           selfHeal: true
-      source:
-        repoURL: {{ _template.repository.url }}
-        targetRevision: HEAD
+      sources:
+      - repoURL: {{ _template.repository.url }}
+        targetRevision: {{ _template.repository.revision }}
         path: {% raw %}'{{ path }}'{% endraw +%}
+        helm:
+          valueFiles:
+          - /values/{% raw %}{{ path.basename }}{% endraw %}/values.yaml
       destination:
         server: {{ _template.cluster.url }}
-        namespace: default
+        namespace: {% raw %}'{{ path[1] }}'{% endraw +%}
diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/serviceaccount.j2.DISABLED b/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/serviceaccount.j2.DISABLED
deleted file mode 100644
index cec2c90..0000000
--- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/serviceaccount.j2.DISABLED
+++ /dev/null
@@ -1,27 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: {{ _template.account.name }}
-  namespace: {{ _template.account.namespace }}
----
-apiVersion: v1
-kind: Secret
-metadata:
-  name: {{ _template.account.name }}-secret
-  namespace: {{ _template.account.namespace }}
-  annotations:
-    kubernetes.io/service-account.name: {{ _template.account.name }}
-type: kubernetes.io/service-account-token
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: {{ _template.clusterrolebinding.name }}
-subjects:
-- kind: ServiceAccount
-  name: {{ _template.account.name }}
-  namespace: {{ _template.account.namespace }}
-roleRef:
-  kind: ClusterRole
-  name: cluster-admin
-  apiGroup: rbac.authorization.k8s.io
diff --git a/ansible/vars/workloadcluster.yml b/ansible/vars/workloadcluster.yml
index 3e090ac..9a09c14 100644
--- a/ansible/vars/workloadcluster.yml
+++ b/ansible/vars/workloadcluster.yml
@@ -11,46 +11,16 @@ downstream:
     longhorn:
       version: 1.4.1
       chart: longhorn/longhorn
+      namespace: longhorn-system
       parse_logic: cat values.yaml | yq eval '.. | select(has("repository")) | .repository + ":" + .tag'
       chart_values: !unsafe |
         defaultSettings:
           defaultDataPath: /mnt/blockstorage
-        global:
-          cattle:
-            systemDefaultRegistry: registry.{{ vapp['metacluster.fqdn'] }}
-        image:
-          longhorn:
-            engine:
-              repository: library/docker.io/longhornio/longhorn-engine
-            manager:
-              repository: library/docker.io/longhornio/longhorn-manager
-            ui:
-              repository: library/docker.io/longhornio/longhorn-ui
-            instanceManager:
-              repository: library/docker.io/longhornio/longhorn-instance-manager
-            shareManager:
-              repository: library/docker.io/longhornio/longhorn-share-manager
-            backingImageManager:
-              repository: library/docker.io/longhornio/backing-image-manager
-            supportBundleKit:
-              repository: library/docker.io/longhornio/support-bundle-kit
-          csi:
-            attacher:
-              repository: library/docker.io/longhornio/csi-attacher
-            provisioner:
-              repository: library/docker.io/longhornio/csi-provisioner
-            nodeDriverRegistrar:
-              repository: library/docker.io/longhornio/csi-node-driver-registrar
-            resizer:
-              repository: library/docker.io/longhornio/csi-resizer
-            snapshotter:
-              repository: library/docker.io/longhornio/csi-snapshotter
-            livenessProbe:
-              repository: library/docker.io/longhornio/livenessprobe
 
     sealed-secrets:
       version: 2.8.1  # (= Sealed Secrets v0.20.2)
       chart: sealed-secrets/sealed-secrets
+      namespace: sealed-secrets
       parse_logic: helm template . | yq --no-doc eval '.. | .image? | select(.)' | sort -u | awk '!/ /'
       chart_values: !unsafe |
         # Empty

From 86a0b684e2aae62c32e3eebde93a20c8e029aac5 Mon Sep 17 00:00:00 2001
From: Danny Bessems <danny@bessems.eu>
Date: Tue, 4 Apr 2023 20:59:54 +0200
Subject: [PATCH 29/51] Add missing key

---
 .../ansible_payload/bootstrap/roles/metacluster/tasks/gitops.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/gitops.yml b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/gitops.yml
index c2799b5..6ff1531 100644
--- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/gitops.yml
+++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/gitops.yml
@@ -61,6 +61,7 @@
             url: https://kubernetes.default.svc
           repository:
             url: https://git.{{ vapp['metacluster.fqdn'] }}/mc/GitOps.Config.git
+            revision: main
       notify:
         - Apply manifests
 

From 979ac38794f5f21bdb10c55fc7d755b9373397a6 Mon Sep 17 00:00:00 2001
From: Danny Bessems <danny@bessems.eu>
Date: Tue, 4 Apr 2023 22:44:56 +0200
Subject: [PATCH 30/51] Aggregate/store workload cluster chart values

---
 ansible/roles/assets/tasks/manifests.yml      | 25 +++++++++++++------
 .../ansible_payload/bootstrap/playbook.yml    |  1 +
 .../roles/workloadcluster/tasks/gitops.yml    |  8 ++++++
 3 files changed, 26 insertions(+), 8 deletions(-)

diff --git a/ansible/roles/assets/tasks/manifests.yml b/ansible/roles/assets/tasks/manifests.yml
index 74240a7..e845e1f 100644
--- a/ansible/roles/assets/tasks/manifests.yml
+++ b/ansible/roles/assets/tasks/manifests.yml
@@ -8,14 +8,6 @@
       loop_control:
         label: "{{ item.key }}"
 
-    # - name: Aggregate chart_values into dict
-    #   ansible.builtin.set_fact:
-    #     workloadcluster_chartvalues: "{{ workloadcluster_chartvalues | default({}) | combine({ (item.key | regex_replace('[^A-Za-z0-9]', '')): { 'chart_values': (item.value.chart_values | from_yaml) } }) }}"
-    #   when: item.value.chart_values is defined
-    #   loop: "{{ lookup('ansible.builtin.dict', downstream.helm_charts) }}"
-    #   loop_control:
-    #     label: "{{ item.key }}"
-
     - name: Write dict to vars_file
       ansible.builtin.copy:
         dest: /opt/firstboot/ansible/vars/metacluster.yml
@@ -28,6 +20,23 @@
             } | to_nice_yaml(indent=2, width=4096)
           }}
 
+    - name: Aggregate chart_values into dict
+      ansible.builtin.set_fact:
+        workloadcluster_chartvalues: "{{ workloadcluster_chartvalues | default({}) | combine({ (item.key | regex_replace('[^A-Za-z0-9]', '')): { 'chart_values': (item.value.chart_values | from_yaml) } }) }}"
+      when: item.value.chart_values is defined
+      loop: "{{ lookup('ansible.builtin.dict', downstream.helm_charts) }}"
+      loop_control:
+        label: "{{ item.key }}"
+
+    - name: Write dict to vars_file
+      ansible.builtin.copy:
+        dest: /opt/firstboot/ansible/vars/workloadcluster.yml
+        content: >-
+          {{
+            { 'downstream_components': ( workloadcluster_chartvalues )
+            } | to_nice_yaml(indent=2, width=4096)
+          }}
+
 - name: Download ClusterAPI manifests
   ansible.builtin.get_url:
     url: "{{ item.url }}"
diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/playbook.yml b/ansible/roles/firstboot/files/ansible_payload/bootstrap/playbook.yml
index 629a28e..02f94aa 100644
--- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/playbook.yml
+++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/playbook.yml
@@ -5,6 +5,7 @@
   vars_files:
     - defaults.yml
     - metacluster.yml
+    - workloadcluster.yml
   # become: true
   roles:
     - vapp
diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/gitops.yml b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/gitops.yml
index 72845ea..b20253a 100644
--- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/gitops.yml
+++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/gitops.yml
@@ -13,6 +13,14 @@
   loop_control:
     label: "{{ item.path | basename }}"
 
+- name: Write chart values to file
+  ansible.builtin.copy:
+    dest: /opt/workloadcluster/git-repositories/gitops/values/{{ item.key }}/values.yaml
+    content: "{{ item.value.chart_values }}"
+  loop: "{{ lookup('ansible.builtin.dict', downstream_components) }}"
+  loop_control:
+    label: "{{ item.key }}"
+
 - name: Initialize/Push git repository
   ansible.builtin.shell:
     cmd: |

From 641ee2d9a7389eb1bf17a616294d2686cfa38b19 Mon Sep 17 00:00:00 2001
From: Danny Bessems <danny@bessems.eu>
Date: Wed, 5 Apr 2023 09:46:05 +0200
Subject: [PATCH 31/51] Rename nodepools

---
 .../kustomization.cluster-template.j2         | 44 +++++++++++++++++++
 1 file changed, 44 insertions(+)

diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/kustomization.cluster-template.j2 b/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/kustomization.cluster-template.j2
index e116fd7..c0aeb3e 100644
--- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/kustomization.cluster-template.j2
+++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/kustomization.cluster-template.j2
@@ -175,3 +175,47 @@ patchesJson6902:
         path: /spec/kubeadmConfigSpec/preKubeadmCommands/-
         value: {{ cmd }}
 {% endfor %}
+
+  - target:
+      group: cluster.x-k8s.io
+      version: v1beta1
+      kind: MachineDeployment
+      name: \${CLUSTER_NAME}-md-0
+    patch: |-
+      - op: replace
+        path: /metadata/name
+        value: ${CLUSTER_NAME}-worker
+      - op: replace
+        path: /spec/template/spec/bootstrap/configRef/name
+        value: ${CLUSTER_NAME}-worker
+  - target:
+      group: bootstrap.cluster.x-k8s.io
+      version: v1beta1
+      kind: KubeadmConfigTemplate
+      name: \${CLUSTER_NAME}-md-0
+    patch: |-
+      - op: replace
+        path: /metadata/name
+        value: ${CLUSTER_NAME}-worker
+
+  - target:
+      group: infrastructure.cluster.x-k8s.io
+      version: v1beta1
+      kind: VSphereMachineTemplate
+      name: \${CLUSTER_NAME}
+    patch: |-
+      - op: replace
+        path: /metadata/name
+        value: ${CLUSTER_NAME}-master
+  - target:
+      group: controlplane.cluster.x-k8s.io
+      version: v1beta1
+      kind: KubeadmControlPlane
+      name: \${CLUSTER_NAME}
+    patch: |-
+      - op: replace
+        path: /metadata/name
+        value: ${CLUSTER_NAME}-master
+      - op: replace
+        path: /spec/machineTemplate/infrastructureRef/name
+        value: ${CLUSTER_NAME}-master

From 2b56677e9a18c0165c88e46be8f930dbbf8abfe0 Mon Sep 17 00:00:00 2001
From: Danny Bessems <danny@bessems.eu>
Date: Wed, 5 Apr 2023 10:47:17 +0200
Subject: [PATCH 32/51] Remove regex_replace filter;Refactor dict key
 names;Make chart values optional

---
 ansible/roles/assets/tasks/manifests.yml               |  4 ++--
 .../bootstrap/roles/metacluster/tasks/git.yml          |  2 +-
 .../bootstrap/roles/metacluster/tasks/gitops.yml       |  2 +-
 .../bootstrap/roles/metacluster/tasks/registry.yml     |  2 +-
 .../bootstrap/roles/metacluster/tasks/storage.yml      |  2 +-
 .../bootstrap/roles/workloadcluster/tasks/gitops.yml   | 10 +++++++++-
 .../upgrade/roles/metacluster/tasks/git.yml            |  2 +-
 .../upgrade/roles/metacluster/tasks/gitops.yml         |  2 +-
 .../upgrade/roles/metacluster/tasks/registry.yml       |  2 +-
 .../upgrade/roles/metacluster/tasks/storage.yml        |  2 +-
 ansible/vars/workloadcluster.yml                       |  4 ++--
 11 files changed, 21 insertions(+), 13 deletions(-)

diff --git a/ansible/roles/assets/tasks/manifests.yml b/ansible/roles/assets/tasks/manifests.yml
index e845e1f..29e13e2 100644
--- a/ansible/roles/assets/tasks/manifests.yml
+++ b/ansible/roles/assets/tasks/manifests.yml
@@ -2,7 +2,7 @@
 
     - name: Aggregate chart_values into dict
       ansible.builtin.set_fact:
-        metacluster_chartvalues: "{{ metacluster_chartvalues | default({}) | combine({ (item.key | regex_replace('[^A-Za-z0-9]', '')): { 'chart_values': (item.value.helm.chart_values | from_yaml) } }) }}"
+        metacluster_chartvalues: "{{ metacluster_chartvalues | default({}) | combine({ item.key: { 'chart_values': (item.value.helm.chart_values | from_yaml) } }) }}"
       when: item.value.helm.chart_values is defined
       loop: "{{ lookup('ansible.builtin.dict', components) }}"
       loop_control:
@@ -22,7 +22,7 @@
 
     - name: Aggregate chart_values into dict
       ansible.builtin.set_fact:
-        workloadcluster_chartvalues: "{{ workloadcluster_chartvalues | default({}) | combine({ (item.key | regex_replace('[^A-Za-z0-9]', '')): { 'chart_values': (item.value.chart_values | from_yaml) } }) }}"
+        workloadcluster_chartvalues: "{{ workloadcluster_chartvalues | default({}) | combine({ item.key: { 'chart_values': (item.value.chart_values | from_yaml) } }) }}"
       when: item.value.chart_values is defined
       loop: "{{ lookup('ansible.builtin.dict', downstream.helm_charts) }}"
       loop_control:
diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/git.yml b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/git.yml
index 85decc5..3da8e60 100644
--- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/git.yml
+++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/git.yml
@@ -8,7 +8,7 @@
         create_namespace: true
         wait: false
         kubeconfig: "{{ kubeconfig.path }}"
-        values: "{{ components.gitea.chart_values }}"
+        values: "{{ components['gitea'].chart_values }}"
 
     - name: Ensure gitea API availability
       ansible.builtin.uri:
diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/gitops.yml b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/gitops.yml
index 6ff1531..5948034 100644
--- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/gitops.yml
+++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/gitops.yml
@@ -8,7 +8,7 @@
         create_namespace: true
         wait: false
         kubeconfig: "{{ kubeconfig.path }}"
-        values: "{{ components.argocd.chart_values }}"
+        values: "{{ components['argo-cd'].chart_values }}"
 
     - name: Ensure argo-cd API availability
       ansible.builtin.uri:
diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/registry.yml b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/registry.yml
index 6589b12..fe4a99d 100644
--- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/registry.yml
+++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/registry.yml
@@ -8,7 +8,7 @@
         create_namespace: true
         wait: false
         kubeconfig: "{{ kubeconfig.path }}"
-        values: "{{ components.harbor.chart_values }}"
+        values: "{{ components['harbor'].chart_values }}"
 
     - name: Ensure harbor API availability
       ansible.builtin.uri:
diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/storage.yml b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/storage.yml
index 059e0a7..17c8f15 100644
--- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/storage.yml
+++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/storage.yml
@@ -7,7 +7,7 @@
         create_namespace: true
         wait: false
         kubeconfig: "{{ kubeconfig.path }}"
-        values: "{{ components.longhorn.chart_values }}"
+        values: "{{ components['longhorn'].chart_values }}"
 
     - name: Ensure longhorn API availability
       ansible.builtin.uri:
diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/gitops.yml b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/gitops.yml
index b20253a..f57e181 100644
--- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/gitops.yml
+++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/gitops.yml
@@ -13,10 +13,18 @@
   loop_control:
     label: "{{ item.path | basename }}"
 
+- name: Create subfolders
+  ansible.builtin.file:
+    path: /opt/workloadcluster/git-repositories/gitops/values/{{ item.key }}
+    state: directory
+  loop: "{{ lookup('ansible.builtin.dict', downstream_components) }}"
+  loop_control:
+    label: "{{ item.key }}"
+
 - name: Write chart values to file
   ansible.builtin.copy:
     dest: /opt/workloadcluster/git-repositories/gitops/values/{{ item.key }}/values.yaml
-    content: "{{ item.value.chart_values }}"
+    content: "{{ item.value.chart_values | default('# Empty') }}"
   loop: "{{ lookup('ansible.builtin.dict', downstream_components) }}"
   loop_control:
     label: "{{ item.key }}"
diff --git a/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/git.yml b/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/git.yml
index 9970549..2454039 100644
--- a/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/git.yml
+++ b/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/git.yml
@@ -7,7 +7,7 @@
         release_namespace: gitea
         wait: false
         kubeconfig: "{{ kubeconfig.path }}"
-        values: "{{ components.gitea.chart_values }}"
+        values: "{{ components['gitea'].chart_values }}"
 
     - name: Ensure gitea API availability
       ansible.builtin.uri:
diff --git a/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/gitops.yml b/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/gitops.yml
index ada716e..79e4ea7 100644
--- a/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/gitops.yml
+++ b/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/gitops.yml
@@ -7,7 +7,7 @@
         release_namespace: argo-cd
         wait: false
         kubeconfig: "{{ kubeconfig.path }}"
-        values: "{{ components.argocd.chart_values }}"
+        values: "{{ components['argo-cd'].chart_values }}"
 
     - name: Ensure argo-cd API availability
       ansible.builtin.uri:
diff --git a/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/registry.yml b/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/registry.yml
index e433a87..9005a2b 100644
--- a/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/registry.yml
+++ b/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/registry.yml
@@ -7,7 +7,7 @@
         release_namespace: harbor
         wait: false
         kubeconfig: "{{ kubeconfig.path }}"
-        values: "{{ components.harbor.chart_values }}"
+        values: "{{ components['harbor'].chart_values }}"
 
     - name: Ensure harbor API availability
       ansible.builtin.uri:
diff --git a/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/storage.yml b/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/storage.yml
index 93861e4..280a6ae 100644
--- a/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/storage.yml
+++ b/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/storage.yml
@@ -34,7 +34,7 @@
         release_namespace: longhorn-system
         wait: false
         kubeconfig: "{{ kubeconfig.path }}"
-        values: "{{ components.longhorn.chart_values }}"
+        values: "{{ components['longhorn'].chart_values }}"
 
     - name: Ensure longhorn API availability
       ansible.builtin.uri:
diff --git a/ansible/vars/workloadcluster.yml b/ansible/vars/workloadcluster.yml
index 9a09c14..99c8f6e 100644
--- a/ansible/vars/workloadcluster.yml
+++ b/ansible/vars/workloadcluster.yml
@@ -22,5 +22,5 @@ downstream:
       chart: sealed-secrets/sealed-secrets
       namespace: sealed-secrets
       parse_logic: helm template . | yq --no-doc eval '.. | .image? | select(.)' | sort -u | awk '!/ /'
-      chart_values: !unsafe |
-        # Empty
+      # chart_values: !unsafe |
+      #   # Empty

From db090ac564528c503340f5429d648812e226051c Mon Sep 17 00:00:00 2001
From: Danny Bessems <danny@bessems.eu>
Date: Wed, 5 Apr 2023 13:37:31 +0200
Subject: [PATCH 33/51] Add missing kustomize patch;Switch to query filter

---
 ansible/roles/assets/tasks/containerimages.yml           | 4 ++--
 ansible/roles/assets/tasks/helm.yml                      | 4 ++--
 ansible/roles/assets/tasks/manifests.yml                 | 6 +++---
 .../bootstrap/roles/workloadcluster/tasks/gitops.yml     | 4 ++--
 .../templates/kustomization.cluster-template.j2          | 9 +++++++++
 .../upgrade/roles/decommission/tasks/storage.yml         | 2 +-
 .../upgrade/roles/metacluster/tasks/storage.yml          | 2 +-
 7 files changed, 20 insertions(+), 11 deletions(-)

diff --git a/ansible/roles/assets/tasks/containerimages.yml b/ansible/roles/assets/tasks/containerimages.yml
index d09f7de..664fce9 100644
--- a/ansible/roles/assets/tasks/containerimages.yml
+++ b/ansible/roles/assets/tasks/containerimages.yml
@@ -20,7 +20,7 @@
     chdir: /opt/metacluster/helm-charts/{{ item.key }}
   register: chartimages_metacluster
   when: item.value.helm is defined
-  loop: "{{ lookup('ansible.builtin.dict', components) }}"
+  loop: "{{ query('ansible.builtin.dict', components) }}"
   loop_control:
     label: "{{ item.key }}"
 
@@ -29,7 +29,7 @@
     cmd: "{{ item.value.parse_logic }}"
     chdir: /opt/workloadcluster/helm-charts/{{ item.value.namespace }}/{{ item.key }}
   register: chartimages_workloadcluster
-  loop: "{{ lookup('ansible.builtin.dict', downstream.helm_charts) }}"
+  loop: "{{ query('ansible.builtin.dict', downstream.helm_charts) }}"
   loop_control:
     label: "{{ item.key }}"
 
diff --git a/ansible/roles/assets/tasks/helm.yml b/ansible/roles/assets/tasks/helm.yml
index 03071c6..8e82710 100644
--- a/ansible/roles/assets/tasks/helm.yml
+++ b/ansible/roles/assets/tasks/helm.yml
@@ -11,7 +11,7 @@
     chdir: /opt/metacluster/helm-charts
   when: item.value.helm is defined
   register: helmcharts_metacluster
-  loop: "{{ lookup('ansible.builtin.dict', components) }}"
+  loop: "{{ query('ansible.builtin.dict', components) }}"
   loop_control:
     label: "{{ item.key }}"
   retries: 5
@@ -23,7 +23,7 @@
     cmd: helm fetch {{ item.value.chart }} --untardir ./{{ item.value.namespace }} --untar --version {{ item.value.version }}
     chdir: /opt/workloadcluster/helm-charts
   register: helmcharts_workloadcluster
-  loop: "{{ lookup('ansible.builtin.dict', downstream.helm_charts) }}"
+  loop: "{{ query('ansible.builtin.dict', downstream.helm_charts) }}"
   loop_control:
     label: "{{ item.key }}"
   retries: 5
diff --git a/ansible/roles/assets/tasks/manifests.yml b/ansible/roles/assets/tasks/manifests.yml
index 29e13e2..5a89f82 100644
--- a/ansible/roles/assets/tasks/manifests.yml
+++ b/ansible/roles/assets/tasks/manifests.yml
@@ -4,7 +4,7 @@
       ansible.builtin.set_fact:
         metacluster_chartvalues: "{{ metacluster_chartvalues | default({}) | combine({ item.key: { 'chart_values': (item.value.helm.chart_values | from_yaml) } }) }}"
       when: item.value.helm.chart_values is defined
-      loop: "{{ lookup('ansible.builtin.dict', components) }}"
+      loop: "{{ query('ansible.builtin.dict', components) }}"
       loop_control:
         label: "{{ item.key }}"
 
@@ -24,7 +24,7 @@
       ansible.builtin.set_fact:
         workloadcluster_chartvalues: "{{ workloadcluster_chartvalues | default({}) | combine({ item.key: { 'chart_values': (item.value.chart_values | from_yaml) } }) }}"
       when: item.value.chart_values is defined
-      loop: "{{ lookup('ansible.builtin.dict', downstream.helm_charts) }}"
+      loop: "{{ query('ansible.builtin.dict', downstream.helm_charts) }}"
       loop_control:
         label: "{{ item.key }}"
 
@@ -104,6 +104,6 @@
 #     owner: root
 #     group: root
 #     mode: 0600
-#   loop: "{{ lookup('ansible.builtin.dict', components) | map(attribute='value.manifests') | list | select('defined') | flatten }}"
+#   loop: "{{ query('ansible.builtin.dict', components) | map(attribute='value.manifests') | list | select('defined') | flatten }}"
 #   loop_control:
 #     label: "{{ item.type ~ '/' ~ item.name }}"
diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/gitops.yml b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/gitops.yml
index f57e181..bc6f7e5 100644
--- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/gitops.yml
+++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/gitops.yml
@@ -17,7 +17,7 @@
   ansible.builtin.file:
     path: /opt/workloadcluster/git-repositories/gitops/values/{{ item.key }}
     state: directory
-  loop: "{{ lookup('ansible.builtin.dict', downstream_components) }}"
+  loop: "{{ query('ansible.builtin.dict', downstream_components) }}"
   loop_control:
     label: "{{ item.key }}"
 
@@ -25,7 +25,7 @@
   ansible.builtin.copy:
     dest: /opt/workloadcluster/git-repositories/gitops/values/{{ item.key }}/values.yaml
     content: "{{ item.value.chart_values | default('# Empty') }}"
-  loop: "{{ lookup('ansible.builtin.dict', downstream_components) }}"
+  loop: "{{ query('ansible.builtin.dict', downstream_components) }}"
   loop_control:
     label: "{{ item.key }}"
 
diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/kustomization.cluster-template.j2 b/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/kustomization.cluster-template.j2
index c0aeb3e..c9c1c18 100644
--- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/kustomization.cluster-template.j2
+++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/kustomization.cluster-template.j2
@@ -219,3 +219,12 @@ patchesJson6902:
       - op: replace
         path: /spec/machineTemplate/infrastructureRef/name
         value: ${CLUSTER_NAME}-master
+  - target:
+      group: cluster.x-k8s.io
+      version: v1beta1
+      kind: Cluster
+      name: \${CLUSTER_NAME}
+    patch: |-
+      - op: replace
+        path: /spec/controlPlaneRef/name
+        value: ${CLUSTER_NAME}-master
diff --git a/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/decommission/tasks/storage.yml b/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/decommission/tasks/storage.yml
index ab3d9c9..a675b83 100644
--- a/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/decommission/tasks/storage.yml
+++ b/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/decommission/tasks/storage.yml
@@ -22,6 +22,6 @@
       spec:
         numberOfReplicas: {{ (lookup('kubernetes.core.k8s', kind='node', kubeconfig=(kubeconfig.path)) | length | int) - 1 }}
     kubeconfig: "{{ kubeconfig.path }}"
-  loop: "{{ lookup('kubernetes.core.k8s', api_version='longhorn.io/v1beta2', kind='volume', namespace='longhorn-system', kubeconfig=(kubeconfig.path)) }}"
+  loop: "{{ query('kubernetes.core.k8s', api_version='longhorn.io/v1beta2', kind='volume', namespace='longhorn-system', kubeconfig=(kubeconfig.path)) }}"
   loop_control:
     label: "{{ item.metadata.name }}"
diff --git a/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/storage.yml b/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/storage.yml
index 280a6ae..282d8a4 100644
--- a/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/storage.yml
+++ b/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/storage.yml
@@ -11,7 +11,7 @@
           spec:
             numberOfReplicas: {{ lookup('kubernetes.core.k8s', kind='node', kubeconfig=(kubeconfig.path)) | length | int }}
         kubeconfig: "{{ kubeconfig.path }}"
-      loop: "{{ lookup('kubernetes.core.k8s', api_version='longhorn.io/v1beta2', kind='volume', namespace='longhorn-system', kubeconfig=(kubeconfig.path)) }}"
+      loop: "{{ query('kubernetes.core.k8s', api_version='longhorn.io/v1beta2', kind='volume', namespace='longhorn-system', kubeconfig=(kubeconfig.path)) }}"
       loop_control:
         label: "{{ item.metadata.name }}"
 

From 4e6a0549b5e3b5022bd9dc142a129806e88cb77d Mon Sep 17 00:00:00 2001
From: Danny Bessems <danny@bessems.eu>
Date: Wed, 5 Apr 2023 16:51:42 +0200
Subject: [PATCH 34/51] Remove redundant task;Refactor packer vm name

---
 .drone.yml                                                  | 4 ++--
 .../upgrade/roles/workloadcluster/tasks/registry.yml        | 6 +-----
 packer/build.pkr.hcl                                        | 4 ++--
 3 files changed, 5 insertions(+), 9 deletions(-)

diff --git a/.drone.yml b/.drone.yml
index f5c154c..7f604db 100644
--- a/.drone.yml
+++ b/.drone.yml
@@ -62,7 +62,7 @@ steps:
       ./packer
   - |
     packer validate \
-      -var vm_name=$DRONE_BUILD_NUMBER-${DRONE_COMMIT_SHA:0:10} \
+      -var vm_name=${DRONE_COMMIT_SHA:0:10}-$(date -d @${DRONE_BUILD_CREATED} +%Y%m%d) \
       -var docker_username=$${DOCKER_USERNAME} \
       -var docker_password=$${DOCKER_PASSWORD} \
       -var repo_username=$${REPO_USERNAME} \
@@ -75,7 +75,7 @@ steps:
     packer build \
       -on-error=cleanup -timestamp-ui \
       -only=vsphere-iso.bootstrap \
-      -var vm_name=$DRONE_BUILD_NUMBER-${DRONE_COMMIT_SHA:0:10} \
+      -var vm_name=${DRONE_COMMIT_SHA:0:10}-$(date -d @${DRONE_BUILD_CREATED} +%Y%m%d) \
       -var docker_username=$${DOCKER_USERNAME} \
       -var docker_password=$${DOCKER_PASSWORD} \
       -var repo_username=$${REPO_USERNAME} \
diff --git a/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/workloadcluster/tasks/registry.yml b/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/workloadcluster/tasks/registry.yml
index 60b1b2b..dcd0285 100644
--- a/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/workloadcluster/tasks/registry.yml
+++ b/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/workloadcluster/tasks/registry.yml
@@ -1,9 +1,5 @@
 - block:
 
-  - name: Lookup kubeadm container images
-    ansible.builtin.set_fact:
-      kubeadm_images: "{{ lookup('ansible.builtin.file', '/opt/metacluster/cluster-api/imagelist').splitlines() }}"
-
   - name: Copy kubeadm container images to dedicated project
     ansible.builtin.uri:
       url: https://registry.{{ vapp['metacluster.fqdn'] }}/api/v2.0/projects/kubeadm/repositories/{{ ( item | regex_findall('([^:/]+)') )[-2] }}/artifacts?from=library/{{ item | replace('/', '%2F') | replace(':', '%3A') }}
@@ -12,7 +8,7 @@
         Authorization: "Basic {{ ('admin:' ~ vapp['metacluster.password']) | b64encode }}"
       body:
         from: "{{ item }}"
-    loop: "{{ kubeadm_images }}"
+    loop: "{{ lookup('ansible.builtin.file', '/opt/metacluster/cluster-api/imagelist').splitlines() }}"
 
   module_defaults:
     ansible.builtin.uri:
diff --git a/packer/build.pkr.hcl b/packer/build.pkr.hcl
index 2c2e4ce..8995dcf 100644
--- a/packer/build.pkr.hcl
+++ b/packer/build.pkr.hcl
@@ -6,7 +6,7 @@ packer {
 build {
   source "vsphere-iso.ubuntu" {
     name = "bootstrap"
-    vm_name = "ova.bootstrap-${var.vm_name}"
+    vm_name = "bld_${var.vm_name}_bootstrap"
 
     export {
       images                = false
@@ -16,7 +16,7 @@ build {
 
   source "vsphere-iso.ubuntu" {
     name = "upgrade"
-    vm_name = "ova.upgrade-${var.vm_name}"
+    vm_name = "bld_${var.vm_name}_upgrade"
 
     export {
       images                = false

From 610495e4247905d6a674df7a799e48c8dbd98c8a Mon Sep 17 00:00:00 2001
From: Danny Bessems <danny@bessems.eu>
Date: Wed, 5 Apr 2023 17:01:36 +0200
Subject: [PATCH 35/51] Add random vm name postfix

---
 .drone.yml | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/.drone.yml b/.drone.yml
index 7f604db..b4b91b8 100644
--- a/.drone.yml
+++ b/.drone.yml
@@ -62,7 +62,8 @@ steps:
       ./packer
   - |
     packer validate \
-      -var vm_name=${DRONE_COMMIT_SHA:0:10}-$(date -d @${DRONE_BUILD_CREATED} +%Y%m%d) \
+      -only=vsphere-iso.bootstrap \
+      -var vm_name=${DRONE_BUILD_NUMBER}-${DRONE_COMMIT_SHA:0:10}-$(openssl rand -hex 3) \
       -var docker_username=$${DOCKER_USERNAME} \
       -var docker_password=$${DOCKER_PASSWORD} \
       -var repo_username=$${REPO_USERNAME} \
@@ -75,7 +76,7 @@ steps:
     packer build \
       -on-error=cleanup -timestamp-ui \
       -only=vsphere-iso.bootstrap \
-      -var vm_name=${DRONE_COMMIT_SHA:0:10}-$(date -d @${DRONE_BUILD_CREATED} +%Y%m%d) \
+      -var vm_name=${DRONE_BUILD_NUMBER}-${DRONE_COMMIT_SHA:0:10}-$(openssl rand -hex 3) \
       -var docker_username=$${DOCKER_USERNAME} \
       -var docker_password=$${DOCKER_PASSWORD} \
       -var repo_username=$${REPO_USERNAME} \
@@ -108,7 +109,7 @@ steps:
   depends_on:
   - Install Ansible Galaxy collections
   image: bv11-cr01.bessems.eu/library/packer-extended
-  pull: always
+  pull: alwaysquery(
   commands:
   - |
     sed -i -e "s/<<img-password>>/$${SSH_PASSWORD}/g" \
@@ -120,7 +121,8 @@ steps:
       ./packer
   - |
     packer validate \
-      -var vm_name=$DRONE_BUILD_NUMBER-${DRONE_COMMIT_SHA:0:10} \
+      -only=vsphere-iso.upgrade \
+      -var vm_name=${DRONE_BUILD_NUMBER}-${DRONE_COMMIT_SHA:0:10}-$(openssl rand -hex 3) \
       -var docker_username=$${DOCKER_USERNAME} \
       -var docker_password=$${DOCKER_PASSWORD} \
       -var repo_username=$${REPO_USERNAME} \
@@ -133,7 +135,7 @@ steps:
     packer build \
       -on-error=cleanup -timestamp-ui \
       -only=vsphere-iso.upgrade \
-      -var vm_name=$DRONE_BUILD_NUMBER-${DRONE_COMMIT_SHA:0:10} \
+      -var vm_name=${DRONE_BUILD_NUMBER}-${DRONE_COMMIT_SHA:0:10}-$(openssl rand -hex 3) \
       -var docker_username=$${DOCKER_USERNAME} \
       -var docker_password=$${DOCKER_PASSWORD} \
       -var repo_username=$${REPO_USERNAME} \

From 1941e02d949adbc2336565a02984e2d8a9d3e9e2 Mon Sep 17 00:00:00 2001
From: Danny Bessems <danny@bessems.eu>
Date: Wed, 5 Apr 2023 17:30:24 +0200
Subject: [PATCH 36/51] Fix post-processor paths

---
 packer/build.pkr.hcl  | 16 +++-------------
 packer/source.pkr.hcl |  5 +++++
 2 files changed, 8 insertions(+), 13 deletions(-)

diff --git a/packer/build.pkr.hcl b/packer/build.pkr.hcl
index 8995dcf..2a1b241 100644
--- a/packer/build.pkr.hcl
+++ b/packer/build.pkr.hcl
@@ -7,21 +7,11 @@ build {
   source "vsphere-iso.ubuntu" {
     name = "bootstrap"
     vm_name = "bld_${var.vm_name}_bootstrap"
-
-    export {
-      images                = false
-      output_directory      = "/scratch/airgapped-k8s/bootstrap"
-    }
   }
 
   source "vsphere-iso.ubuntu" {
     name = "upgrade"
     vm_name = "bld_${var.vm_name}_upgrade"
-
-    export {
-      images                = false
-      output_directory      = "/scratch/airgapped-k8s/upgrade"
-    }
   }
 
   provisioner "ansible" {
@@ -50,11 +40,11 @@ build {
     inline = [
       "pwsh -command \"& scripts/Update-OvfConfiguration.ps1 \\",
       " -ApplianceType '${source.name}' \\",
-      " -OVFFile '/scratch/airgapped-k8s/${source.name}/ova.${source.name}-${var.vm_name}.ovf' \"",
+      " -OVFFile '/scratch/bld_${var.vm_name}_${source.name}.ovf' \"",
       "pwsh -file scripts/Update-Manifest.ps1 \\",
-      " -ManifestFileName '/scratch/airgapped-k8s/${source.name}/ova.${source.name}-${var.vm_name}.mf'",
+      " -ManifestFileName '/scratch/bld_${var.vm_name}_${source.name}.mf'",
       "ovftool --acceptAllEulas --allowExtraConfig --overwrite \\",
-      " '/scratch/airgapped-k8s/${source.name}/ova.${source.name}-${var.vm_name}.ovf' \\",
+      " '/scratch/bld_${var.vm_name}_${source.name}.ovf' \\",
       " /output/airgapped-k8s-${var.k8s_version}.${source.name}.ova"
     ]
   }
diff --git a/packer/source.pkr.hcl b/packer/source.pkr.hcl
index 3774c76..02b2f3f 100644
--- a/packer/source.pkr.hcl
+++ b/packer/source.pkr.hcl
@@ -53,4 +53,9 @@ source "vsphere-iso" "ubuntu" {
   shutdown_timeout        = "5m"
 
   remove_cdrom            = true
+
+  export {
+    images                = false
+    output_directory      = "/scratch"
+  }
 }

From 5c18869d60ea9fd381b4f91c6a48f5e80a60db3a Mon Sep 17 00:00:00 2001
From: Danny Bessems <danny@bessems.eu>
Date: Thu, 6 Apr 2023 08:47:37 +0200
Subject: [PATCH 37/51] Fix missing namespaces;Add default empty value

---
 ansible/roles/assets/tasks/manifests.yml                      | 4 ++--
 .../ansible_payload/bootstrap/templates/applicationset.j2     | 2 ++
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/ansible/roles/assets/tasks/manifests.yml b/ansible/roles/assets/tasks/manifests.yml
index 5a89f82..52068fb 100644
--- a/ansible/roles/assets/tasks/manifests.yml
+++ b/ansible/roles/assets/tasks/manifests.yml
@@ -22,8 +22,8 @@
 
     - name: Aggregate chart_values into dict
       ansible.builtin.set_fact:
-        workloadcluster_chartvalues: "{{ workloadcluster_chartvalues | default({}) | combine({ item.key: { 'chart_values': (item.value.chart_values | from_yaml) } }) }}"
-      when: item.value.chart_values is defined
+        workloadcluster_chartvalues: "{{ workloadcluster_chartvalues | default({}) | combine({ item.key: { 'chart_values': (item.value.chart_values | default('') | from_yaml) } }) }}"
+      # when: item.value.chart_values is defined
       loop: "{{ query('ansible.builtin.dict', downstream.helm_charts) }}"
       loop_control:
         label: "{{ item.key }}"
diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/applicationset.j2 b/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/applicationset.j2
index cdfd461..580752f 100644
--- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/applicationset.j2
+++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/applicationset.j2
@@ -19,6 +19,8 @@ spec:
         automated:
           prune: true
           selfHeal: true
+        syncOptions:
+        - CreateNamespace=true
       sources:
       - repoURL: {{ _template.repository.url }}
         targetRevision: {{ _template.repository.revision }}

From c385baf630d5e2f4b9f3bc360ddeb5b59376572b Mon Sep 17 00:00:00 2001
From: Danny Bessems <danny@bessems.eu>
Date: Thu, 6 Apr 2023 13:29:29 +0200
Subject: [PATCH 38/51] Housekeeping;Add separate storage nodepool

---
 .../roles/metacluster/tasks/gitops.yml        |  2 +-
 .../workloadcluster/tasks/clusterapi.yml      | 27 +++++--
 .../roles/workloadcluster/tasks/gitops.yml    |  2 +-
 .../kustomization.cluster-template.j2         | 58 ++++++---------
 .../templates/kustomization.nodepool.j2       | 74 +++++++++++++++++++
 scripts/Update-OvfConfiguration.bootstrap.yml | 36 ++++-----
 6 files changed, 138 insertions(+), 61 deletions(-)
 create mode 100644 ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/kustomization.nodepool.j2

diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/gitops.yml b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/gitops.yml
index 5948034..8157a00 100644
--- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/gitops.yml
+++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/gitops.yml
@@ -39,7 +39,7 @@
         mode: 0600
       vars:
         _template:
-          name: argocd-gitrepo-mc-gitopsconfig
+          name: gitrepo-mc-gitopsconfig
           namespace: argo-cd
           url: https://git.{{ vapp['metacluster.fqdn'] }}/mc/GitOps.Config.git
       notify:
diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/clusterapi.yml b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/clusterapi.yml
index 360e138..70e6e0b 100644
--- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/clusterapi.yml
+++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/clusterapi.yml
@@ -52,13 +52,12 @@
         version: "{{ components.clusterapi.workload.version.k8s }}"
         vip: "{{ vapp['workloadcluster.vip'] }}"
 
-- name: Generate kustomization template
+- name: Generate cluster-template kustomization manifest
   ansible.builtin.template:
     src: kustomization.cluster-template.j2
     dest: /opt/metacluster/cluster-api/infrastructure-vsphere/{{ components.clusterapi.management.version.infrastructure_vsphere }}/kustomization.yaml
   vars:
     _template:
-      additionaldisk: "{{ vapp['workloadcluster.additionaldisk'] }}"
       network:
         fqdn: "{{ vapp['metacluster.fqdn'] }}"
         dnsserver: "{{ vapp['guestinfo.dnsserver'] }}"
@@ -105,7 +104,8 @@
     clustersize: >-
       {{ {
         'controlplane': vapp['deployment.type'] | regex_findall('^cp(\d)+') | first,
-        'workers': vapp['deployment.type'] | regex_findall('w(\d)+$') | first
+        'worker': vapp['deployment.type'] | regex_findall('w(\d)+$') | first,
+        'workerstorage': vapp['deployment.type'] | regex_findall('ws(\d)+') | first
       } }}
 
 - name: Generate workload cluster manifest
@@ -114,7 +114,7 @@
       clusterctl generate cluster \
         {{ vapp['workloadcluster.name'] | lower }} \
         --control-plane-machine-count {{ clustersize.controlplane }} \
-        --worker-machine-count {{ clustersize.workers }} \
+        --worker-machine-count {{ clustersize.worker }} \
         --from ./custom-cluster-template.yaml \
         --config ./clusterctl.yaml \
         --kubeconfig {{ kubeconfig.path }}
@@ -138,6 +138,23 @@
         -f {{ capi_clustermanifest.path }}/new-cluster.yaml \
         -o {{ capi_clustermanifest.path }}/manifests
 
+- name: Generate nodepool kustomization manifest
+  ansible.builtin.template:
+    src: kustomization.nodepool.j2
+    dest: "{{ capi_clustermanifest.path }}/kustomization.yaml"
+  vars:
+    _template:
+      cluster:
+        name: "{{ vapp['workloadcluster.name'] }}"
+      nodepool:
+        size: "{{ clustersize.workerstorage }}"
+        additionaldisk: "{{ vapp['workloadcluster.additionaldisk'] }}"
+
+- name: Store nodepool manifest
+  ansible.builtin.copy:
+    dest: "{{ capi_clustermanifest.path }}/manifests/nodepool-worker-storage.yaml"
+    content: "{{ lookup('kubernetes.core.kustomize', dir=capi_clustermanifest.path) }}"
+
 - name: Create in-cluster IpPool
   ansible.builtin.template:
     src: ippool.j2
@@ -181,7 +198,7 @@
     mode: 0600
   vars:
     _template:
-      name: argocd-gitrepo-mc-gitopsclusterapi
+      name: gitrepo-mc-gitopsclusterapi
       namespace: argo-cd
       url: https://git.{{ vapp['metacluster.fqdn'] }}/mc/GitOps.ClusterAPI.git
   notify:
diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/gitops.yml b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/gitops.yml
index bc6f7e5..31e5585 100644
--- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/gitops.yml
+++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/gitops.yml
@@ -75,7 +75,7 @@
     mode: 0600
   vars:
     _template:
-      name: argocd-gitrepo-wl-gitopsconfig
+      name: gitrepo-wl-gitopsconfig
       namespace: argo-cd
       url: https://git.{{ vapp['metacluster.fqdn'] }}/wl/GitOps.Config.git
   notify:
diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/kustomization.cluster-template.j2 b/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/kustomization.cluster-template.j2
index c9c1c18..afb747d 100644
--- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/kustomization.cluster-template.j2
+++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/kustomization.cluster-template.j2
@@ -25,18 +25,6 @@ patchesStrategicMerge:
         spec:
           clusterConfiguration:
             imageRepository: registry.{{ _template.network.fqdn }}/kubeadm
-          diskSetup:
-            filesystems:
-            - device: /dev/sdb1
-              filesystem: ext4
-              label: blockstorage
-            partitions:
-            - device: /dev/sdb
-              layout: true
-              tableType: gpt
-          mounts:
-          - - LABEL=blockstorage
-            - /mnt/blockstorage
   - |-
     apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
     kind: KubeadmConfigTemplate
@@ -98,8 +86,6 @@ patchesStrategicMerge:
     spec:
       template:
         spec:
-          additionalDisksGiB:
-          - {{ _template.additionaldisk }}
           network:
             devices:
             - dhcp4: false
@@ -176,28 +162,6 @@ patchesJson6902:
         value: {{ cmd }}
 {% endfor %}
 
-  - target:
-      group: cluster.x-k8s.io
-      version: v1beta1
-      kind: MachineDeployment
-      name: \${CLUSTER_NAME}-md-0
-    patch: |-
-      - op: replace
-        path: /metadata/name
-        value: ${CLUSTER_NAME}-worker
-      - op: replace
-        path: /spec/template/spec/bootstrap/configRef/name
-        value: ${CLUSTER_NAME}-worker
-  - target:
-      group: bootstrap.cluster.x-k8s.io
-      version: v1beta1
-      kind: KubeadmConfigTemplate
-      name: \${CLUSTER_NAME}-md-0
-    patch: |-
-      - op: replace
-        path: /metadata/name
-        value: ${CLUSTER_NAME}-worker
-
   - target:
       group: infrastructure.cluster.x-k8s.io
       version: v1beta1
@@ -228,3 +192,25 @@ patchesJson6902:
       - op: replace
         path: /spec/controlPlaneRef/name
         value: ${CLUSTER_NAME}-master
+
+  - target:
+      group: cluster.x-k8s.io
+      version: v1beta1
+      kind: MachineDeployment
+      name: \${CLUSTER_NAME}-md-0
+    patch: |-
+      - op: replace
+        path: /metadata/name
+        value: ${CLUSTER_NAME}-worker
+      - op: replace
+        path: /spec/template/spec/bootstrap/configRef/name
+        value: ${CLUSTER_NAME}-worker
+  - target:
+      group: bootstrap.cluster.x-k8s.io
+      version: v1beta1
+      kind: KubeadmConfigTemplate
+      name: \${CLUSTER_NAME}-md-0
+    patch: |-
+      - op: replace
+        path: /metadata/name
+        value: ${CLUSTER_NAME}-worker
diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/kustomization.nodepool.j2 b/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/kustomization.nodepool.j2
new file mode 100644
index 0000000..9cd2a33
--- /dev/null
+++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/kustomization.nodepool.j2
@@ -0,0 +1,74 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- manifests/kubeadmconfigtemplate-{{ _template.cluster.name }}-worker.yaml
+- manifests/machinedeployment-{{ _template.cluster.name }}-worker.yaml
+- manifests/vspheremachinetemplate-{{ _template.cluster.name }}-worker.yaml
+
+patchesStrategicMerge:
+  - |-
+    apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
+    kind: KubeadmConfigTemplate
+    metadata:
+      name: {{ _template.cluster.name }}-worker
+      namespace: default
+    spec:
+      template:
+        spec:
+          diskSetup:
+            filesystems:
+            - device: /dev/sdb1
+              filesystem: ext4
+              label: blockstorage
+            partitions:
+            - device: /dev/sdb
+              layout: true
+              tableType: gpt
+          mounts:
+          - - LABEL=blockstorage
+            - /mnt/blockstorage
+
+patchesJson6902:
+  - target:
+      group: bootstrap.cluster.x-k8s.io
+      version: v1beta1
+      kind: KubeadmConfigTemplate
+      name: {{ _template.cluster.name }}-worker
+    patch: |-
+      - op: replace
+        path: /metadata/name
+        value: {{ _template.cluster.name }}-worker-storage
+
+  - target:
+      group: cluster.x-k8s.io
+      version: v1beta1
+      kind: MachineDeployment
+      name: {{ _template.cluster.name }}-worker
+    patch: |-
+      - op: replace
+        path: /metadata/name
+        value: {{ _template.cluster.name }}-worker-storage
+      - op: replace
+        path: /spec/template/spec/bootstrap/configRef/name
+        value: {{ _template.cluster.name }}-worker-storage
+      - op: replace
+        path: /spec/template/spec/infrastructureRef/name
+        value: {{ _template.cluster.name }}-worker-storage
+      - op: replace
+        path: /spec/replicas
+        value: {{ _template.nodepool.size }}
+
+  - target:
+      group: infrastructure.cluster.x-k8s.io
+      version: v1beta1
+      kind: VSphereMachineTemplate
+      name: {{ _template.cluster.name }}-worker
+    patch: |-
+      - op: replace
+        path: /metadata/name
+        value: {{ _template.cluster.name }}-worker-storage
+      - op: add
+        path: /spec/template/spec
+        value:
+          additionalDisksGiB:
+          - {{ _template.nodepool.additionaldisk }}
diff --git a/scripts/Update-OvfConfiguration.bootstrap.yml b/scripts/Update-OvfConfiguration.bootstrap.yml
index 3204984..9514fe5 100644
--- a/scripts/Update-OvfConfiguration.bootstrap.yml
+++ b/scripts/Update-OvfConfiguration.bootstrap.yml
@@ -1,11 +1,11 @@
 DeploymentConfigurations:
 
-- Id: cp1w1
+- Id: cp1w1ws0
   Label: 'Workload-cluster: 1 control-plane node/1 worker node'
   Description: 1 control-plane node/1 worker node
 
-- Id: cp1w2
-  Label: 'Workload-cluster: 1 control-plane node/2 worker nodes'
+- Id: cp1w1ws1
+  Label: 'Workload-cluster: 1 control-plane node/1 worker node/1 worker-storage node'
   Description: 1 control-plane node/2 worker nodes
 
 - Id: core
@@ -28,8 +28,8 @@ PropertyCategories:
   - Key: deployment.type
     Type: string
     Value:
-    - cp1w1
-    - cp1w2
+    - cp1w1ws0
+    - cp1w1ws1
     - core
     UserConfigurable: false
 
@@ -128,8 +128,8 @@ PropertyCategories:
     Description: ''
     DefaultValue: 'workload-{{ hostname.suffix }}'
     Configurations:
-    - cp1w1
-    - cp1w2
+    - cp1w1ws0
+    - cp1w1ws1
     UserConfigurable: true
 
   - Key: workloadcluster.vip
@@ -138,8 +138,8 @@ PropertyCategories:
     Description: Workload-cluster control plane endpoint virtual IP address
     DefaultValue: '0.0.0.0'
     Configurations:
-    - cp1w1
-    - cp1w2
+    - cp1w1ws0
+    - cp1w1ws1
     UserConfigurable: true
 
   - Key: ippool.startip
@@ -148,8 +148,8 @@ PropertyCategories:
     Description: All nodes for the workload-cluster will be provisioned within this IP pool
     DefaultValue: '0.0.0.0'
     Configurations:
-    - cp1w1
-    - cp1w2
+    - cp1w1ws0
+    - cp1w1ws1
     UserConfigurable: true
 
   - Key: ippool.endip
@@ -158,8 +158,8 @@ PropertyCategories:
     Description: All nodes for the workload-cluster will be provisioned within this IP pool
     DefaultValue: '0.0.0.0'
     Configurations:
-    - cp1w1
-    - cp1w2
+    - cp1w1ws0
+    - cp1w1ws1
     UserConfigurable: true
 
   - Key: workloadcluster.nodesize
@@ -171,18 +171,18 @@ PropertyCategories:
       - MEDIUM: 4 vCPU/8GB RAM
     DefaultValue: 'small'
     Configurations:
-    - cp1w1
-    - cp1w2
+    - cp1w1ws0
+    - cp1w1ws1
     UserConfigurable: true
 
   - Key: workloadcluster.additionaldisk
     Type: int(0..120)
     Label: Workload-cluster block storage disk size*
-    Description: 'All worker-nodes for the workload-cluster will be provisioned with an additional disk of the specified size'
+    Description: 'All worker-storage nodes for the workload-cluster will be provisioned with an additional disk of the specified size'
     DefaultValue: '20'
     Configurations:
-    - cp1w1
-    - cp1w2
+    - cp1w1ws0
+    - cp1w1ws1
     UserConfigurable: true
 
 - Name: 4) Common

From 2f91c0f7c3ef46c417d77c6d1e4d0da9cc39a491 Mon Sep 17 00:00:00 2001
From: Danny Bessems <danny@bessems.eu>
Date: Fri, 7 Apr 2023 10:29:41 +0200
Subject: [PATCH 39/51] Move kustomize pattern to strategic merge;Fix regex
 patterns;Update description

---
 .../roles/workloadcluster/tasks/clusterapi.yml   |  4 ++--
 .../templates/kustomization.nodepool.j2          | 16 +++++++++++-----
 scripts/Update-OvfConfiguration.bootstrap.yml    |  2 +-
 3 files changed, 14 insertions(+), 8 deletions(-)

diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/clusterapi.yml b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/clusterapi.yml
index 70e6e0b..75830cb 100644
--- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/clusterapi.yml
+++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/clusterapi.yml
@@ -104,8 +104,8 @@
     clustersize: >-
       {{ {
         'controlplane': vapp['deployment.type'] | regex_findall('^cp(\d)+') | first,
-        'worker': vapp['deployment.type'] | regex_findall('w(\d)+$') | first,
-        'workerstorage': vapp['deployment.type'] | regex_findall('ws(\d)+') | first
+        'worker': vapp['deployment.type'] | regex_findall('w(\d)+') | first,
+        'workerstorage': vapp['deployment.type'] | regex_findall('ws(\d)+$') | first
       } }}
 
 - name: Generate workload cluster manifest
diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/kustomization.nodepool.j2 b/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/kustomization.nodepool.j2
index 9cd2a33..8a1dd95 100644
--- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/kustomization.nodepool.j2
+++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/kustomization.nodepool.j2
@@ -27,6 +27,17 @@ patchesStrategicMerge:
           mounts:
           - - LABEL=blockstorage
             - /mnt/blockstorage
+  - |-
+    apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+    kind: VSphereMachineTemplate
+    metadata:
+      name: {{ _template.cluster.name }}-worker
+      namespace: default
+    spec:
+      template:
+        spec:
+          additionalDisksGiB:
+          - {{ _template.nodepool.additionaldisk }}
 
 patchesJson6902:
   - target:
@@ -67,8 +78,3 @@ patchesJson6902:
       - op: replace
         path: /metadata/name
         value: {{ _template.cluster.name }}-worker-storage
-      - op: add
-        path: /spec/template/spec
-        value:
-          additionalDisksGiB:
-          - {{ _template.nodepool.additionaldisk }}
diff --git a/scripts/Update-OvfConfiguration.bootstrap.yml b/scripts/Update-OvfConfiguration.bootstrap.yml
index 9514fe5..12f068c 100644
--- a/scripts/Update-OvfConfiguration.bootstrap.yml
+++ b/scripts/Update-OvfConfiguration.bootstrap.yml
@@ -6,7 +6,7 @@ DeploymentConfigurations:
 
 - Id: cp1w1ws1
   Label: 'Workload-cluster: 1 control-plane node/1 worker node/1 worker-storage node'
-  Description: 1 control-plane node/2 worker nodes
+  Description: 1 control-plane node/1 worker node/1 worker-storage node
 
 - Id: core
   Label: No workload-cluster

From 05c3a09ab322f98c13530f94a8e4bf5f1d2f9ff0 Mon Sep 17 00:00:00 2001
From: Danny Bessems <danny@bessems.eu>
Date: Fri, 7 Apr 2023 11:59:29 +0200
Subject: [PATCH 40/51] Upgrade k3s version

---
 ansible/vars/metacluster.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ansible/vars/metacluster.yml b/ansible/vars/metacluster.yml
index 1862fc7..1e77e4d 100644
--- a/ansible/vars/metacluster.yml
+++ b/ansible/vars/metacluster.yml
@@ -1,7 +1,7 @@
 platform:
 
   k3s:
-    version: v1.26.2+k3s1
+    version: v1.26.3+k3s1
 
   # gitops:
   #   repository:

From 2534cea4a0dbdaf8ed090e5bc6a237130b04c3b1 Mon Sep 17 00:00:00 2001
From: Danny Bessems <danny@bessems.eu>
Date: Fri, 7 Apr 2023 13:14:34 +0200
Subject: [PATCH 41/51] Pin k3s install.sh version

---
 ansible/roles/assets/tasks/k3s.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ansible/roles/assets/tasks/k3s.yml b/ansible/roles/assets/tasks/k3s.yml
index 5da958f..abe1f12 100644
--- a/ansible/roles/assets/tasks/k3s.yml
+++ b/ansible/roles/assets/tasks/k3s.yml
@@ -21,7 +21,7 @@
 
 - name: Download K3s install script
   ansible.builtin.get_url:
-    url: https://get.k3s.io
+    url: https://github.com/k3s-io/k3s/blob/{{ platform.k3s.version }}/install.sh
     dest: /opt/metacluster/k3s/install.sh
     owner: root
     group: root

From debe80a2a18891c95b006e7c82292e151914d453 Mon Sep 17 00:00:00 2001
From: Danny Bessems <danny@bessems.eu>
Date: Fri, 7 Apr 2023 13:19:57 +0200
Subject: [PATCH 42/51] Fix url

---
 ansible/roles/assets/tasks/k3s.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ansible/roles/assets/tasks/k3s.yml b/ansible/roles/assets/tasks/k3s.yml
index abe1f12..7b4c2c1 100644
--- a/ansible/roles/assets/tasks/k3s.yml
+++ b/ansible/roles/assets/tasks/k3s.yml
@@ -21,7 +21,7 @@
 
 - name: Download K3s install script
   ansible.builtin.get_url:
-    url: https://github.com/k3s-io/k3s/blob/{{ platform.k3s.version }}/install.sh
+    url: https://raw.githubusercontent.com/k3s-io/k3s/{{ platform.k3s.version | urlencode }}/install.sh
     dest: /opt/metacluster/k3s/install.sh
     owner: root
     group: root

From 75277e285afb20821ee40892dc6b99dce8e06982 Mon Sep 17 00:00:00 2001
From: Danny Bessems <danny@bessems.eu>
Date: Sun, 9 Apr 2023 20:04:11 +0200
Subject: [PATCH 43/51] Switch oidc provider

---
 ansible/vars/metacluster.yml | 118 +++++++++++++++++++++--------------
 1 file changed, 71 insertions(+), 47 deletions(-)

diff --git a/ansible/vars/metacluster.yml b/ansible/vars/metacluster.yml
index 1e77e4d..01ef0cc 100644
--- a/ansible/vars/metacluster.yml
+++ b/ansible/vars/metacluster.yml
@@ -39,8 +39,10 @@ platform:
   helm_repositories:
     - name: argo
       url: https://argoproj.github.io/argo-helm
-    - name: dex
-      url: https://charts.dexidp.io
+    - name: codecentric
+      url: https://codecentric.github.io/helm-charts
+    # - name: dex
+    #   url: https://charts.dexidp.io
     - name: gitea-charts
       url: https://dl.gitea.io/charts/
     - name: harbor
@@ -99,53 +101,53 @@ components:
       node_template:
         url: https://{{ repo_username }}:{{ repo_password }}@sn.itch.fyi/Repository/rel/ubuntu-2004-kube-v1.26.3.ova
 
-  dex:
-    helm:
-      version: 0.13.0 # (= Dex 2.35.3)
-      chart: dex/dex
-      parse_logic: helm template . | yq --no-doc eval '.. | .image? | select(.)' | sort -u | awk '!/ /'
-      chart_values: !unsafe |
-        config:
-          connectors:
-            - type: ldap
-              id: ldap
-              name: "LDAP"
-              config:
-                host: "{{ vapp['ldap.fqdn'] }}:636"
-                insecureNoSSL: false
-                insecureSkipVerify: true
-                bindDN: "{{ vapp['ldap.dn'] }}"
-                bindPW: "{{ vapp['ldap.password'] }}"
+  # dex:
+  #   helm:
+  #     version: 0.13.0 # (= Dex 2.35.3)
+  #     chart: dex/dex
+  #     parse_logic: helm template . | yq --no-doc eval '.. | .image? | select(.)' | sort -u | awk '!/ /'
+  #     chart_values: !unsafe |
+  #       config:
+  #         connectors:
+  #           - type: ldap
+  #             id: ldap
+  #             name: "LDAP"
+  #             config:
+  #               host: "{{ vapp['ldap.fqdn'] }}:636"
+  #               insecureNoSSL: false
+  #               insecureSkipVerify: true
+  #               bindDN: "{{ vapp['ldap.dn'] }}"
+  #               bindPW: "{{ vapp['ldap.password'] }}"
 
-                usernamePrompt: "Username"
-                userSearch:
-                  baseDN: OU=Administrators,OU=Useraccounts,DC=bessems,DC=eu
-                  filter: "(objectClass=person)"
-                  username: userPrincipalName
-                  idAttr: DN
-                  emailAttr: userPrincipalName
-                  nameAttr: cn
+  #               usernamePrompt: "Username"
+  #               userSearch:
+  #                 baseDN: OU=Administrators,OU=Useraccounts,DC=bessems,DC=eu
+  #                 filter: "(objectClass=person)"
+  #                 username: userPrincipalName
+  #                 idAttr: DN
+  #                 emailAttr: userPrincipalName
+  #                 nameAttr: cn
 
-                groupSearch:
-                  baseDN: OU=Roles,OU=Groups,DC=bessems,DC=eu
-                  filter: "(objectClass=group)"
-                  userMatchers:
-                  - userAttr: DN
-                    groupAttr: member
-                  nameAttr: cn
-          enablePasswordDB: true
-          issuer: https://oidc.{{ vapp['metacluster.fqdn'] }}
-          storage:
-            type: kubernetes
-            config:
-              inCluster: true
-        ingress:
-          enabled: true
-          hosts:
-            - host: oidc.{{ vapp['metacluster.fqdn'] }}
-              paths:
-                - path: /
-                  pathType: Prefix
+  #               groupSearch:
+  #                 baseDN: OU=Roles,OU=Groups,DC=bessems,DC=eu
+  #                 filter: "(objectClass=group)"
+  #                 userMatchers:
+  #                 - userAttr: DN
+  #                   groupAttr: member
+  #                 nameAttr: cn
+  #         enablePasswordDB: true
+  #         issuer: https://oidc.{{ vapp['metacluster.fqdn'] }}
+  #         storage:
+  #           type: kubernetes
+  #           config:
+  #             inCluster: true
+  #       ingress:
+  #         enabled: true
+  #         hosts:
+  #           - host: oidc.{{ vapp['metacluster.fqdn'] }}
+  #             paths:
+  #               - path: /
+  #                 pathType: Prefix
 
   gitea:
     helm:
@@ -201,6 +203,28 @@ components:
             registry:
               size: 25Gi
 
+  keycloakx:
+    helm:
+      version: 2.1.1  # (= Keycloak 20.0.3)
+      chart: codecentric/keycloakx
+      parse_logic: helm template . | yq --no-doc eval '.. | .image? | select(.)' | sort -u | awk '!/ /'
+      chart_values: !unsafe |
+        command:
+          - "/opt/keycloak/bin/kc.sh"
+          - "start"
+          - "--http-enabled=true"
+          - "--http-port=8080"
+          - "--hostname-strict=false"
+          - "--hostname-strict-https=false"
+        extraEnv: |
+          - name: KEYCLOAK_ADMIN
+            value: admin
+          - name: KEYCLOAK_ADMIN_PASSWORD
+            value: {{ vapp['metacluster.password'] }}
+          - name: JAVA_OPTS_APPEND
+            value: >-
+              -Djgroups.dns.query={{ include "keycloak.fullname" . }}-headless
+
   kube-prometheus-stack:
     helm:
       version: 45.2.0

From ee08fd47b5468244fb01950e782c3e522941f44a Mon Sep 17 00:00:00 2001
From: Danny Bessems <danny@bessems.eu>
Date: Tue, 11 Apr 2023 10:40:33 +0200
Subject: [PATCH 44/51] Configure keycloakx;Convert output to yaml

---
 .../bootstrap/roles/workloadcluster/tasks/gitops.yml |  2 +-
 ansible/vars/metacluster.yml                         | 12 +++++++++++-
 2 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/gitops.yml b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/gitops.yml
index 31e5585..e426700 100644
--- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/gitops.yml
+++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/gitops.yml
@@ -24,7 +24,7 @@
 - name: Write chart values to file
   ansible.builtin.copy:
     dest: /opt/workloadcluster/git-repositories/gitops/values/{{ item.key }}/values.yaml
-    content: "{{ item.value.chart_values | default('# Empty') }}"
+    content: "{{ item.value.chart_values | default('# Empty') | to_nice_yaml(indent=2, width=4096) }}"
   loop: "{{ query('ansible.builtin.dict', downstream_components) }}"
   loop_control:
     label: "{{ item.key }}"
diff --git a/ansible/vars/metacluster.yml b/ansible/vars/metacluster.yml
index 01ef0cc..7a71c83 100644
--- a/ansible/vars/metacluster.yml
+++ b/ansible/vars/metacluster.yml
@@ -221,9 +221,19 @@ components:
             value: admin
           - name: KEYCLOAK_ADMIN_PASSWORD
             value: {{ vapp['metacluster.password'] }}
+          - name: KC_PROXY
+            value: "passthrough"
           - name: JAVA_OPTS_APPEND
             value: >-
-              -Djgroups.dns.query={{ include "keycloak.fullname" . }}-headless
+              -Djgroups.dns.query={% raw %}{{ include "keycloak.fullname" . }}{% endraw %}-headless
+        ingress:
+          enabled: true
+          rules:
+            - host: keycloak.{{ vapp['metacluster.fqdn'] }}
+              paths:
+                - path: /
+                  pathType: Prefix
+          tls: []
 
   kube-prometheus-stack:
     helm:

From 572b7df74cae7fd2b37d67982ac967e8766abc90 Mon Sep 17 00:00:00 2001
From: Danny Bessems <danny@bessems.eu>
Date: Thu, 13 Apr 2023 10:11:13 +0200
Subject: [PATCH 45/51] Switch OIDC provider

---
 ansible/vars/metacluster.yml | 97 +++++++++++++++++++++---------------
 1 file changed, 58 insertions(+), 39 deletions(-)

diff --git a/ansible/vars/metacluster.yml b/ansible/vars/metacluster.yml
index 7a71c83..a772e01 100644
--- a/ansible/vars/metacluster.yml
+++ b/ansible/vars/metacluster.yml
@@ -3,12 +3,6 @@ platform:
   k3s:
     version: v1.26.3+k3s1
 
-  # gitops:
-  #   repository:
-  #     uri: https://code.spamasaurus.com/djpbessems/GitOps.MetaCluster.git
-  #     # revision: v0.1.0
-  #     revision: HEAD
-
   packaged_components:
     - name: traefik
       namespace: kube-system
@@ -39,8 +33,10 @@ platform:
   helm_repositories:
     - name: argo
       url: https://argoproj.github.io/argo-helm
-    - name: codecentric
-      url: https://codecentric.github.io/helm-charts
+    - name: authentik
+      url: https://charts.goauthentik.io
+    # - name: codecentric
+    #   url: https://codecentric.github.io/helm-charts
     # - name: dex
     #   url: https://charts.dexidp.io
     - name: gitea-charts
@@ -75,6 +71,29 @@ components:
             hosts:
               - gitops.{{ vapp['metacluster.fqdn'] }}
 
+  authentik:
+    helm:
+      version: 2023.3.1
+      chart: authentik/authentik
+      parse_logic: helm template . --set postgresql.enabled=true,redis.enabled=true | yq --no-doc eval '.. | .image? | select(.)' | sort -u | awk '!/ /'
+      chart_values: !unsafe |
+        authentik:
+          secret_key: {{ lookup('ansible.builtin.password', '/dev/null length=64 chars=ascii_lowercase,digits seed=' ~ vapp['guestinfo.hostname']) }}
+          postgresql:
+            password: {{ lookup('ansible.builtin.password', '/dev/null length=32 chars=ascii_lowercase,digits seed=' ~ vapp['guestinfo.hostname']) }}
+        ingress:
+          enabled: true
+          hosts:
+            - host: auth.{{ vapp['metacluster.fqdn'] }}
+              paths:
+                - path: "/"
+                  pathType: Prefix
+        postgresql:
+          enabled: true
+          postgresqlPassword: {{ lookup('ansible.builtin.password', '/dev/null length=32 chars=ascii_lowercase,digits seed=' ~ vapp['guestinfo.hostname']) }}
+        redis:
+          enabled: true
+
   cert-manager:
     helm:
       version: 1.11.0
@@ -203,37 +222,37 @@ components:
             registry:
               size: 25Gi
 
-  keycloakx:
-    helm:
-      version: 2.1.1  # (= Keycloak 20.0.3)
-      chart: codecentric/keycloakx
-      parse_logic: helm template . | yq --no-doc eval '.. | .image? | select(.)' | sort -u | awk '!/ /'
-      chart_values: !unsafe |
-        command:
-          - "/opt/keycloak/bin/kc.sh"
-          - "start"
-          - "--http-enabled=true"
-          - "--http-port=8080"
-          - "--hostname-strict=false"
-          - "--hostname-strict-https=false"
-        extraEnv: |
-          - name: KEYCLOAK_ADMIN
-            value: admin
-          - name: KEYCLOAK_ADMIN_PASSWORD
-            value: {{ vapp['metacluster.password'] }}
-          - name: KC_PROXY
-            value: "passthrough"
-          - name: JAVA_OPTS_APPEND
-            value: >-
-              -Djgroups.dns.query={% raw %}{{ include "keycloak.fullname" . }}{% endraw %}-headless
-        ingress:
-          enabled: true
-          rules:
-            - host: keycloak.{{ vapp['metacluster.fqdn'] }}
-              paths:
-                - path: /
-                  pathType: Prefix
-          tls: []
+  # keycloakx:
+  #   helm:
+  #     version: 2.1.1  # (= Keycloak 20.0.3)
+  #     chart: codecentric/keycloakx
+  #     parse_logic: helm template . | yq --no-doc eval '.. | .image? | select(.)' | sort -u | awk '!/ /'
+  #     chart_values: !unsafe |
+  #       command:
+  #         - "/opt/keycloak/bin/kc.sh"
+  #         - "start"
+  #         - "--http-enabled=true"
+  #         - "--http-port=8080"
+  #         - "--hostname-strict=false"
+  #         - "--hostname-strict-https=false"
+  #       extraEnv: |
+  #         - name: KEYCLOAK_ADMIN
+  #           value: admin
+  #         - name: KEYCLOAK_ADMIN_PASSWORD
+  #           value: {{ vapp['metacluster.password'] }}
+  #         - name: KC_PROXY
+  #           value: "passthrough"
+  #         - name: JAVA_OPTS_APPEND
+  #           value: >-
+  #             -Djgroups.dns.query={% raw %}{{ include "keycloak.fullname" . }}{% endraw %}-headless
+  #       ingress:
+  #         enabled: true
+  #         rules:
+  #           - host: keycloak.{{ vapp['metacluster.fqdn'] }}
+  #             paths:
+  #               - path: /
+  #                 pathType: Prefix
+  #         tls: []
 
   kube-prometheus-stack:
     helm:

From d73320da3256c3f69f7a48b6f707e07d534a65dc Mon Sep 17 00:00:00 2001
From: Danny Bessems <danny@bessems.eu>
Date: Thu, 13 Apr 2023 10:47:52 +0200
Subject: [PATCH 46/51] Add quotes

---
 ansible/vars/metacluster.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/ansible/vars/metacluster.yml b/ansible/vars/metacluster.yml
index a772e01..b84471c 100644
--- a/ansible/vars/metacluster.yml
+++ b/ansible/vars/metacluster.yml
@@ -78,9 +78,9 @@ components:
       parse_logic: helm template . --set postgresql.enabled=true,redis.enabled=true | yq --no-doc eval '.. | .image? | select(.)' | sort -u | awk '!/ /'
       chart_values: !unsafe |
         authentik:
-          secret_key: {{ lookup('ansible.builtin.password', '/dev/null length=64 chars=ascii_lowercase,digits seed=' ~ vapp['guestinfo.hostname']) }}
+          secret_key: "{{ lookup('ansible.builtin.password', '/dev/null length=64 chars=ascii_lowercase,digits seed=' ~ vapp['guestinfo.hostname']) }}"
           postgresql:
-            password: {{ lookup('ansible.builtin.password', '/dev/null length=32 chars=ascii_lowercase,digits seed=' ~ vapp['guestinfo.hostname']) }}
+            password: "{{ lookup('ansible.builtin.password', '/dev/null length=32 chars=ascii_lowercase,digits seed=' ~ vapp['guestinfo.hostname']) }}"
         ingress:
           enabled: true
           hosts:
@@ -90,7 +90,7 @@ components:
                   pathType: Prefix
         postgresql:
           enabled: true
-          postgresqlPassword: {{ lookup('ansible.builtin.password', '/dev/null length=32 chars=ascii_lowercase,digits seed=' ~ vapp['guestinfo.hostname']) }}
+          postgresqlPassword: "{{ lookup('ansible.builtin.password', '/dev/null length=32 chars=ascii_lowercase,digits seed=' ~ vapp['guestinfo.hostname']) }}"
         redis:
           enabled: true
 

From 16fdd66328e2322aca36c915a2eb4c3cd7c41a2e Mon Sep 17 00:00:00 2001
From: Danny Bessems <danny@bessems.eu>
Date: Fri, 14 Apr 2023 09:51:59 +0200
Subject: [PATCH 47/51] Hide redundant parameter;Configure oidc provider

---
 ansible/vars/metacluster.yml                  | 3 +++
 scripts/Update-OvfConfiguration.bootstrap.yml | 3 +--
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/ansible/vars/metacluster.yml b/ansible/vars/metacluster.yml
index b84471c..d01e1dd 100644
--- a/ansible/vars/metacluster.yml
+++ b/ansible/vars/metacluster.yml
@@ -78,9 +78,12 @@ components:
       parse_logic: helm template . --set postgresql.enabled=true,redis.enabled=true | yq --no-doc eval '.. | .image? | select(.)' | sort -u | awk '!/ /'
       chart_values: !unsafe |
         authentik:
+          avatars: none
           secret_key: "{{ lookup('ansible.builtin.password', '/dev/null length=64 chars=ascii_lowercase,digits seed=' ~ vapp['guestinfo.hostname']) }}"
           postgresql:
             password: "{{ lookup('ansible.builtin.password', '/dev/null length=32 chars=ascii_lowercase,digits seed=' ~ vapp['guestinfo.hostname']) }}"
+        env:
+          AUTHENTIK_BOOTSTRAP_PASSWORD: "{{ vapp['metacluster.password'] }}"
         ingress:
           enabled: true
           hosts:
diff --git a/scripts/Update-OvfConfiguration.bootstrap.yml b/scripts/Update-OvfConfiguration.bootstrap.yml
index 12f068c..635dfea 100644
--- a/scripts/Update-OvfConfiguration.bootstrap.yml
+++ b/scripts/Update-OvfConfiguration.bootstrap.yml
@@ -179,9 +179,8 @@ PropertyCategories:
     Type: int(0..120)
     Label: Workload-cluster block storage disk size*
     Description: 'All worker-storage nodes for the workload-cluster will be provisioned with an additional disk of the specified size'
-    DefaultValue: '20'
+    DefaultValue: '42'
     Configurations:
-    - cp1w1ws0
     - cp1w1ws1
     UserConfigurable: true
 

From 1860d8e2dd557e703d8643ccc6f45b43317b6208 Mon Sep 17 00:00:00 2001
From: Danny Bessems <danny@bessems.eu>
Date: Tue, 18 Apr 2023 21:50:26 +0200
Subject: [PATCH 48/51] Configure longhorn through node label;Update version

---
 .../bootstrap/templates/kustomization.nodepool.j2             | 4 ++++
 ansible/vars/metacluster.yml                                  | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/kustomization.nodepool.j2 b/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/kustomization.nodepool.j2
index 8a1dd95..eff0edc 100644
--- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/kustomization.nodepool.j2
+++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/kustomization.nodepool.j2
@@ -24,6 +24,10 @@ patchesStrategicMerge:
             - device: /dev/sdb
               layout: true
               tableType: gpt
+          joinConfiguration:
+            nodeRegistration:
+              kubeletExtraArgs:
+                node-labels: "node.longhorn.io/create-default-disk=true"
           mounts:
           - - LABEL=blockstorage
             - /mnt/blockstorage
diff --git a/ansible/vars/metacluster.yml b/ansible/vars/metacluster.yml
index d01e1dd..69a13fa 100644
--- a/ansible/vars/metacluster.yml
+++ b/ansible/vars/metacluster.yml
@@ -346,7 +346,7 @@ dependencies:
       url: https://github.com/patrickdappollonio/kubectl-slice/releases/download/v1.2.5/kubectl-slice_linux_x86_64.tar.gz
       archive: compressed
     - filename: skopeo
-      url: https://code.spamasaurus.com/api/packages/djpbessems/generic/skopeo/v1.11.1/skopeo_linux_amd64
+      url: https://code.spamasaurus.com/api/packages/djpbessems/generic/skopeo/v1.12.0/skopeo_linux_amd64
     - filename: step
       url: https://dl.step.sm/gh-release/cli/gh-release-header/v0.23.0/step_linux_0.23.0_amd64.tar.gz
       archive: compressed

From 3c98e16e741d5574f7966a5c7afbe19889f9702b Mon Sep 17 00:00:00 2001
From: Danny Bessems <danny@bessems.eu>
Date: Wed, 19 Apr 2023 08:43:34 +0200
Subject: [PATCH 49/51] Update longhorn settings

---
 ansible/vars/workloadcluster.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ansible/vars/workloadcluster.yml b/ansible/vars/workloadcluster.yml
index 99c8f6e..c4b15d6 100644
--- a/ansible/vars/workloadcluster.yml
+++ b/ansible/vars/workloadcluster.yml
@@ -15,6 +15,7 @@ downstream:
       parse_logic: cat values.yaml | yq eval '.. | select(has("repository")) | .repository + ":" + .tag'
       chart_values: !unsafe |
         defaultSettings:
+          createDefaultDiskLabeledNodes: true
           defaultDataPath: /mnt/blockstorage
 
     sealed-secrets:

From ca8044b4ab6dee1ca4bbf1079c53ad86878b0c9d Mon Sep 17 00:00:00 2001
From: Danny Bessems <danny@bessems.eu>
Date: Tue, 25 Apr 2023 09:01:10 +0200
Subject: [PATCH 50/51] Workaround to support self-signed vCenter certificate

---
 .../kustomization.cluster-template.j2         | 28 +++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/kustomization.cluster-template.j2 b/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/kustomization.cluster-template.j2
index afb747d..d7734bb 100644
--- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/kustomization.cluster-template.j2
+++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/kustomization.cluster-template.j2
@@ -4,6 +4,34 @@ resources:
 - cluster-template.yaml
 
 patchesStrategicMerge:
+  - |-
+    apiVersion: v1
+    kind: Secret
+    metadata:
+      name: csi-vsphere-config
+      namespace: '${NAMESPACE}'
+    stringData:
+      data: |
+        apiVersion: v1
+        kind: Secret
+        metadata:
+          name: csi-vsphere-config
+          namespace: kube-system
+        stringData:
+          csi-vsphere.conf: |+
+            [Global]
+            insecure-flag = true
+            thumbprint = "${VSPHERE_TLS_THUMBPRINT}"
+            cluster-id = "${NAMESPACE}/${CLUSTER_NAME}"
+
+            [VirtualCenter "${VSPHERE_SERVER}"]
+            user = "${VSPHERE_USERNAME}"
+            password = "${VSPHERE_PASSWORD}"
+            datacenters = "${VSPHERE_DATACENTER}"
+
+            [Network]
+            public-network = "${VSPHERE_NETWORK}"
+        type: Opaque
   - |-
     apiVersion: controlplane.cluster.x-k8s.io/v1beta1
     kind: KubeadmControlPlane

From fd1c3060615fb9082db57e1f4ee436c742d63586 Mon Sep 17 00:00:00 2001
From: Danny Bessems <danny@bessems.eu>
Date: Wed, 26 Apr 2023 09:17:41 +0200
Subject: [PATCH 51/51] Add workload-cluster worker-node size property

---
 .../roles/workloadcluster/tasks/clusterapi.yml      |  3 +++
 .../templates/kustomization.cluster-template.j2     | 12 ++++++++++++
 .../files/ansible_payload/common/vars/defaults.yml  | 13 +++++++++++++
 scripts/Update-OvfConfiguration.bootstrap.yml       |  5 +++--
 4 files changed, 31 insertions(+), 2 deletions(-)

diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/clusterapi.yml b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/clusterapi.yml
index 75830cb..575fe15 100644
--- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/clusterapi.yml
+++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/clusterapi.yml
@@ -61,6 +61,9 @@
       network:
         fqdn: "{{ vapp['metacluster.fqdn'] }}"
         dnsserver: "{{ vapp['guestinfo.dnsserver'] }}"
+      nodesize:
+        cpu: "{{ config.clusterapi.size_matrix[ vapp['workloadcluster.nodesize'] ].cpu }}"
+        memory: "{{ config.clusterapi.size_matrix[ vapp['workloadcluster.nodesize'] ].memory }}"
       rootca: "{{ stepca_cm_certs.resources[0].data['root_ca.crt'] }}"
       runcmds:
         - update-ca-certificates
diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/kustomization.cluster-template.j2 b/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/kustomization.cluster-template.j2
index d7734bb..f273815 100644
--- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/kustomization.cluster-template.j2
+++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/kustomization.cluster-template.j2
@@ -221,6 +221,18 @@ patchesJson6902:
         path: /spec/controlPlaneRef/name
         value: ${CLUSTER_NAME}-master
 
+  - target:
+      group: infrastructure.cluster.x-k8s.io
+      version: v1beta1
+      kind: VSphereMachineTemplate
+      name: \${CLUSTER_NAME}-worker
+    patch: |-
+      - op: replace
+        path: /spec/template/spec/numCPUs
+        value: {{ _template.nodesize.cpu }}
+      - op: replace
+        path: /spec/template/spec/memoryMiB
+        value: {{ _template.nodesize.memory }}
   - target:
       group: cluster.x-k8s.io
       version: v1beta1
diff --git a/ansible/roles/firstboot/files/ansible_payload/common/vars/defaults.yml b/ansible/roles/firstboot/files/ansible_payload/common/vars/defaults.yml
index c3b1df9..3d2b32c 100644
--- a/ansible/roles/firstboot/files/ansible_payload/common/vars/defaults.yml
+++ b/ansible/roles/firstboot/files/ansible_payload/common/vars/defaults.yml
@@ -8,3 +8,16 @@ playbook:
 
 # This default value is updated during the playbook, based on an I/O intensive operation
 storage_benchmark: 30
+
+config:
+  clusterapi:
+    size_matrix:
+      small:
+        cpu: 2
+        memory: 6144
+      medium:
+        cpu: 4
+        memory: 8192
+      large:
+        cpu: 8
+        memory: 16384
diff --git a/scripts/Update-OvfConfiguration.bootstrap.yml b/scripts/Update-OvfConfiguration.bootstrap.yml
index 635dfea..8089d71 100644
--- a/scripts/Update-OvfConfiguration.bootstrap.yml
+++ b/scripts/Update-OvfConfiguration.bootstrap.yml
@@ -163,12 +163,13 @@ PropertyCategories:
     UserConfigurable: true
 
   - Key: workloadcluster.nodesize
-    Type: string["small", "medium"]
+    Type: string["small", "medium", "large"]
     Label: Workload-cluster node size*
     Description: |
-      All worker-nodes for the workload-cluster will be provisioned with number of cpu-cores and memory as specified:
+      All worker and worker-storage nodes for the workload-cluster will be provisioned with number of cpu-cores and memory as specified:
       - SMALL: 2 vCPU/6GB RAM
       - MEDIUM: 4 vCPU/8GB RAM
+      - LARGE: 8 vCPU/16GB RAM
     DefaultValue: 'small'
     Configurations:
     - cp1w1ws0