fix: Generate and store kubeconfig in repository

.drone.yml

@@ -26,8 +26,6 @@ steps:
   - yamllint --version
 
 - name: Linting
-  depends_on:
-  - Debugging information
   image: bv11-cr01.bessems.eu/library/packer-extended
   pull: always
   commands:
@@ -38,8 +36,6 @@ steps:
       scripts
 
 - name: Semantic Release (Dry-run)
-  depends_on:
-  - Linting
   image: bv11-cr01.bessems.eu/proxy/library/node:20-slim
   pull: always
   commands:
@@ -47,21 +43,29 @@ steps:
     apt-get update
   - |
     apt-get install -y --no-install-recommends \
+      curl \
       git-core \
+      jq \
       ca-certificates
+  - |
+    curl -L https://api.github.com/repos/mikefarah/yq/releases/latest | \
+      jq -r '.assets[] | select(.name | endswith("yq_linux_amd64")) | .browser_download_url' | \
+      xargs -I {} curl -L -o /bin/yq {} && \
+    chmod +x /bin/yq
   - |
     npm install \
       semantic-release \
       @semantic-release/commit-analyzer \
       @semantic-release/exec \
   - |
+    export K8S_VERSION=$(yq '.components.clusterapi.workload.version.k8s' < ./ansible/vars/metacluster.yml)
     export GIT_CREDENTIALS=$${GIT_USERNAME}:$${GIT_APIKEY}
   - |
     npx semantic-release \
       --package @semantic-release/exec \
       --package semantic-release \
       --branches ${DRONE_BRANCH} \
-      --tag-format "K8s_1.25.9-v\$${version}" \
+      --tag-format "K8s_$${K8S_VERSION}-v\$${version}" \
       --dry-run \
       --plugins @semantic-release/commit-analyzer,@semantic-release/exec \
       --analyzeCommits @semantic-release/commit-analyzer \
@@ -73,8 +77,6 @@ steps:
     GIT_USERNAME: djpbessems
 
 - name: Install Ansible Galaxy collections
-  depends_on:
-  - Semantic Release (Dry-run)
   image: bv11-cr01.bessems.eu/library/packer-extended
   pull: always
   commands:
@@ -84,8 +86,6 @@ steps:
       -p ./ansible/collections
 
 - name: Kubernetes Bootstrap Appliance
-  depends_on:
-  - Install Ansible Galaxy collections
   image: bv11-cr01.bessems.eu/library/packer-extended
   pull: always
   commands:
@@ -94,7 +94,7 @@ steps:
       packer/preseed/UbuntuServer22.04/user-data
   - |
     export K8S_VERSION=$(yq '.components.clusterapi.workload.version.k8s' < ./ansible/vars/metacluster.yml)
-    export NEXT_RELEASE_VERSION=$(cat .version)
+    export APPLIANCE_VERSION=$(cat .version)
   - |
     packer init -upgrade \
       ./packer
@@ -109,7 +109,7 @@ steps:
       -var ssh_password=$${SSH_PASSWORD} \
       -var vsphere_password=$${VSPHERE_PASSWORD} \
       -var k8s_version=$K8S_VERSION \
-      -var next_release_version=$NEXT_RELEASE_VERSION \
+      -var appliance_version=$APPLIANCE_VERSION \
       ./packer
   - |
     packer build \
@@ -123,7 +123,7 @@ steps:
       -var ssh_password=$${SSH_PASSWORD} \
       -var vsphere_password=$${VSPHERE_PASSWORD} \
       -var k8s_version=$K8S_VERSION \
-      -var next_release_version=$NEXT_RELEASE_VERSION \
+      -var appliance_version=$APPLIANCE_VERSION \
       ./packer
   environment:
     DOCKER_USERNAME:
@@ -146,8 +146,6 @@ steps:
     path: /scratch
 
 - name: Kubernetes Upgrade Appliance
-  depends_on:
-  - Install Ansible Galaxy collections
   image: bv11-cr01.bessems.eu/library/packer-extended
   pull: alwaysquery(
   commands:
@@ -156,7 +154,7 @@ steps:
       packer/preseed/UbuntuServer22.04/user-data
   - |
     export K8S_VERSION=$(yq '.components.clusterapi.workload.version.k8s' < ./ansible/vars/metacluster.yml)
-    export NEXT_RELEASE_VERSION=$(cat .version)
+    export APPLIANCE_VERSION=$(cat .version)
   - |
     packer init -upgrade \
       ./packer
@@ -171,7 +169,7 @@ steps:
       -var ssh_password=$${SSH_PASSWORD} \
       -var vsphere_password=$${VSPHERE_PASSWORD} \
       -var k8s_version=$K8S_VERSION \
-      -var next_release_version=$NEXT_RELEASE_VERSION \
+      -var appliance_version=$APPLIANCE_VERSION \
       ./packer
   - |
     packer build \
@@ -185,7 +183,7 @@ steps:
       -var ssh_password=$${SSH_PASSWORD} \
       -var vsphere_password=$${VSPHERE_PASSWORD} \
       -var k8s_version=$K8S_VERSION \
-      -var next_release_version=$NEXT_RELEASE_VERSION \
+      -var appliance_version=$APPLIANCE_VERSION \
       ./packer
   environment:
     DOCKER_USERNAME:
@@ -208,9 +206,6 @@ steps:
     path: /scratch
 
 - name: Remove temporary resources
-  depends_on:
-  - Kubernetes Bootstrap Appliance
-  - Kubernetes Upgrade Appliance
   image: bv11-cr01.bessems.eu/library/packer-extended
   commands:
   - |

.releaserc.json.DISABLED

@@ -1,11 +0,0 @@
-{
-    "plugins": [
-        ["@semantic-release/commit-analyzer"],
-        ["@semantic-release/release-notes-generator"],
-        ["@semantic-release/exec", {
-            "prepareCmd": "export SEMANTICRELEASE_NEXTRELEASEVERSION=${nextRelease.version}",
-            "publishCmd": "echo $SEMANTICRELEASE_NEXTRELEASEVERSION"
-        }],
-        ["@semantic-release/git"]
-    ]
-}

ansible/roles/assets/tasks/containerimages.yml

@@ -1,4 +1,4 @@
-- name: Parse manifests for container images
+- name: Parse Cluster-API manifests for container images
   ansible.builtin.shell:
     # This set of commands is necessary to deal with multi-line scalar values
     # eg.:
@@ -9,7 +9,7 @@
       cat {{ item.dest }} | yq --no-doc eval '.. | .image? | select(.)' | awk '!/ /';
       cat {{ item.dest }} | yq eval '.data.data' | yq --no-doc eval '.. | .image? | select(.)';
       cat {{ item.dest }} | yq --no-doc eval '.. | .files? | with_entries(select(.value.path == "*.yaml")).[0].content' | awk '!/null/' | yq eval '.. | .image? | select(.)'
-  register: parsedmanifests
+  register: clusterapi_parsedmanifests
   loop: "{{ clusterapi_manifests.results }}"
   loop_control:
     label: "{{ item.dest | basename }}"
@@ -41,8 +41,8 @@
       results: "{{ (chartimages_metacluster | json_query('results[*].stdout_lines')) + (chartimages_workloadcluster | json_query('results[*].stdout_lines')) | select() | flatten | list }}"
     - source: kubeadm
       results: "{{ kubeadmimages.stdout_lines }}"
-    - source: manifests
-      results: "{{ parsedmanifests | json_query('results[*].stdout_lines') | select() | flatten | list }}"
+    - source: clusterapi
+      results: "{{ clusterapi_parsedmanifests | json_query('results[*].stdout_lines') | select() | flatten | list }}"
   loop_control:
     label: "{{ item.source }}"
 
@@ -64,4 +64,4 @@
         docker://{{ item }} \
         docker-archive:./{{ ( item | regex_findall('[^/:]+'))[-2] }}_{{ lookup('ansible.builtin.password', '/dev/null length=5 chars=ascii_lowercase,digits seed={{ item }}') }}.tar:{{ item }}
     chdir: /opt/metacluster/container-images
-  loop: "{{ (containerimages_charts + containerimages_kubeadm + containerimages_manifests + dependencies.container_images) | flatten | unique | sort }}"
+  loop: "{{ (containerimages_charts + containerimages_kubeadm + containerimages_clusterapi + dependencies.container_images) | flatten | unique | sort }}"

ansible/roles/assets/tasks/main.yml

@@ -16,8 +16,6 @@
     - /opt/metacluster/helm-charts
     - /opt/metacluster/k3s
     - /opt/metacluster/kube-vip
-    - /opt/workloadcluster/git-repositories/gitops/charts
-    - /opt/workloadcluster/git-repositories/gitops/values
     - /opt/workloadcluster/helm-charts
     - /opt/workloadcluster/node-templates
     - /var/lib/rancher/k3s/agent/images

ansible/roles/assets/tasks/manifests.yml

@@ -1,6 +1,6 @@
 - block:
 
-    - name: Aggregate chart_values into dict
+    - name: Aggregate meta-cluster chart_values into dict
       ansible.builtin.set_fact:
         metacluster_chartvalues: "{{ metacluster_chartvalues | default({}) | combine({ item.key: { 'chart_values': (item.value.helm.chart_values | from_yaml) } }) }}"
       when: item.value.helm.chart_values is defined
@@ -8,7 +8,7 @@
       loop_control:
         label: "{{ item.key }}"
 
-    - name: Write dict to vars_file
+    - name: Combine and write dict to vars_file
       ansible.builtin.copy:
         dest: /opt/firstboot/ansible/vars/metacluster.yml
         content: >-
@@ -16,14 +16,25 @@
             { 'components': (
               metacluster_chartvalues |
               combine({ 'clusterapi': components.clusterapi }) |
-              combine({ 'kubevip'   : components.kubevip }) )
+              combine({ 'kubevip'   : components.kubevip }) ),
+              'appliance': {
+                'version': (applianceversion)
+              }
             } | to_nice_yaml(indent=2, width=4096)
           }}
 
-    - name: Aggregate chart_values into dict
+    - name: Aggregate workload-cluster chart_values into dict
       ansible.builtin.set_fact:
-        workloadcluster_chartvalues: "{{ workloadcluster_chartvalues | default({}) | combine({ item.key: { 'chart_values': (item.value.chart_values | default('') | from_yaml) } }) }}"
-      # when: item.value.chart_values is defined
+        workloadcluster_chartvalues: |
+          {{
+            workloadcluster_chartvalues | default({}) | combine({
+              item.key: {
+                'chart_values': (item.value.chart_values | default('') | from_yaml),
+                'extra_manifests': (item.value.extra_manifests | default([])),
+                'namespace': (item.value.namespace)
+              }
+            })
+          }}
       loop: "{{ query('ansible.builtin.dict', downstream.helm_charts) }}"
       loop_control:
         label: "{{ item.key }}"
@@ -37,7 +48,7 @@
             } | to_nice_yaml(indent=2, width=4096)
           }}
 
-- name: Download ClusterAPI manifests
+- name: Download Cluster-API manifests
   ansible.builtin.get_url:
     url: "{{ item.url }}"
     dest: /opt/metacluster/cluster-api/{{ item.dest }}

ansible/roles/firstboot/files/ansible_payload/bootstrap/playbook.yml

@@ -2,6 +2,9 @@
 - hosts: 127.0.0.1
   connection: local
   gather_facts: true
+  vars:
+    # Needed by some templating in various tasks
+    _newline: "\n"
   vars_files:
     - defaults.yml
     - metacluster.yml

ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/authentication.yml

@@ -0,0 +1,151 @@
+- block:
+
+    - name: Install dex
+      kubernetes.core.helm:
+        name: dex
+        chart_ref: /opt/metacluster/helm-charts/dex
+        release_namespace: dex
+        create_namespace: true
+        wait: false
+        kubeconfig: "{{ kubeconfig.path }}"
+        values: "{{ components['dex'].chart_values }}"
+
+- block:
+
+    - name: Install pinniped chart
+      kubernetes.core.helm:
+        name: pinniped
+        chart_ref: /opt/metacluster/helm-charts/pinniped
+        release_namespace: pinniped-supervisor
+        create_namespace: true
+        wait: false
+        kubeconfig: "{{ kubeconfig.path }}"
+        values: "{{ components['pinniped'].chart_values }}"
+
+    - name: Add ingress for supervisor
+      kubernetes.core.k8s:
+        template: "{{ item.kind }}.j2"
+        state: present
+        kubeconfig: "{{ kubeconfig.path }}"
+      vars:
+        _template:
+          name: "{{ item.name }}"
+          namespace: "{{ item.namespace }}"
+          spec: "{{ item.spec }}"
+      loop:
+        - kind: ingressroute
+          name: pinniped-supervisor
+          namespace: pinniped-supervisor
+          spec: |2
+              entryPoints:
+              - web
+              - websecure
+              routes:
+              - kind: Rule
+                match: Host(`auth.{{ vapp['metacluster.fqdn'] }}`)
+                services:
+                - kind: Service
+                  name: pinniped-supervisor
+                  namespace: pinniped-supervisor
+                  port: 443
+                  scheme: https
+                  serversTransport: pinniped-supervisor
+        - kind: serverstransport
+          name: pinniped-supervisor
+          namespace: pinniped-supervisor
+          spec: |2
+              insecureSkipVerify: true
+              serverName: auth.{{ vapp['metacluster.fqdn'] }}
+      loop_control:
+        label: "{{ item.kind ~ '/' ~ item.name ~ ' (' ~ item.namespace ~ ')' }}"
+
+    - name: Ensure pinniped API availability
+      ansible.builtin.uri:
+        url: https://auth.{{ vapp['metacluster.fqdn'] }}/healthz
+        method: GET
+      register: api_readycheck
+      until:
+        - api_readycheck.status == 200
+        - api_readycheck.msg is search("OK")
+      retries: "{{ playbook.retries }}"
+      delay: "{{ ((storage_benchmark | float) * playbook.delay.short) | int }}"
+
+    # TODO: Migrate to step-ca
+    - name: Initialize tempfile
+      ansible.builtin.tempfile:
+        state: directory
+      register: certificate
+
+    - name: Create private key (RSA, 4096 bits)
+      community.crypto.openssl_privatekey:
+        path: "{{ certificate.path }}/certificate.key"
+
+    - name: Create self-signed certificate
+      community.crypto.x509_certificate:
+        path: "{{ certificate.path }}/certificate.crt"
+        privatekey_path: "{{ certificate.path }}/certificate.key"
+        provider: selfsigned
+
+    - name: Store self-signed certificate for use by pinniped supervisor
+      kubernetes.core.k8s:
+        template: secret.j2
+        state: present
+        kubeconfig: "{{ kubeconfig.path }}"
+      vars:
+        _template:
+          name: pinniped-supervisor-tls
+          namespace: pinniped-supervisor
+          type: kubernetes.io/tls
+          data:
+            - key: tls.crt
+              value: "{{ lookup('ansible.builtin.file', certificate.path ~ '/certificate.crt') | b64encode }}"
+            - key: tls.key
+              value: "{{ lookup('ansible.builtin.file', certificate.path ~ '/certificate.key') | b64encode }}"
+    # TODO: Migrate to step-ca
+
+    - name: Create pinniped resources
+      kubernetes.core.k8s:
+        template: "{{ item.kind }}.j2"
+        state: present
+        kubeconfig: "{{ kubeconfig.path }}"
+      vars:
+        _template:
+          name: "{{ item.name }}"
+          namespace: "{{ item.namespace }}"
+          type: "{{ item.type | default('') }}"
+          data: "{{ item.data | default(omit) }}"
+          spec: "{{ item.spec | default(omit) }}"
+      loop:
+        - kind: oidcidentityprovider
+          name: dex-staticpasswords
+          namespace: pinniped-supervisor
+          spec: |2
+              issuer: https://idps.{{ vapp['metacluster.fqdn'] }}
+              tls:
+                certificateAuthorityData: "{{ (stepca_cm_certs.resources[0].data['intermediate_ca.crt'] ~ _newline ~ stepca_cm_certs.resources[0].data['root_ca.crt']) | b64encode }}"
+              authorizationConfig:
+                additionalScopes: [offline_access, groups, email]
+                allowPasswordGrant: false
+              claims:
+                username: email
+                groups: groups
+              client:
+                secretName: dex-clientcredentials
+        - kind: secret
+          name: dex-clientcredentials
+          namespace: pinniped-supervisor
+          type: secrets.pinniped.dev/oidc-client
+          data:
+            - key: clientID
+              value: "{{ 'pinniped-supervisor' | b64encode }}"
+            - key: clientSecret
+              value: "{{ lookup('ansible.builtin.password', '/dev/null length=64 chars=ascii_lowercase,digits seed=' ~ vapp['metacluster.fqdn']) | b64encode }}"
+        - kind: federationdomain
+          name: metacluster-sso
+          namespace: pinniped-supervisor
+          spec: |2
+              issuer: https://auth.{{ vapp['metacluster.fqdn'] }}/sso
+              tls:
+                secretName: pinniped-supervisor-tls
+      loop_control:
+        label: "{{ item.kind ~ '/' ~ item.name }}"

ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/certauthority.yml

@@ -1,14 +1,9 @@
 - block:
 
-    - name: Initialize tempfile
-      ansible.builtin.tempfile:
-        state: file
-      register: values_file
-
-    - name: Write chart values w/ password to tempfile
+    - name: Inject password into values file
       ansible.builtin.copy:
-        dest: "{{ values_file.path }}"
-        content: "{{ stepca_values.stdout | regex_replace('(ca_password|provisioner_password): ', '\\1: ' ~ (vapp['metacluster.password'] | b64encode)) }}"
+        dest: "{{ stepconfig.path }}"
+        content: "{{ lookup('ansible.builtin.file', stepconfig.path) | regex_replace('(ca_password|provisioner_password):[ ]?\n', '\\1: ' ~ (vapp['metacluster.password'] | b64encode) ~ '\n') }}"
       no_log: true
 
     - name: Install step-ca chart
@@ -21,13 +16,7 @@
         wait: true
         kubeconfig: "{{ kubeconfig.path }}"
         values_files:
-          - "{{ values_file.path }}"
-
-    - name: Cleanup tempfile
-      ansible.builtin.file:
-        path: "{{ values_file.path }}"
-        state: absent
-      when: values_file.path is defined
+          - "{{ stepconfig.path }}"
 
     - name: Retrieve configmap w/ root certificate
       kubernetes.core.k8s_info:
@@ -58,6 +47,7 @@
           namespace: "{{ item.namespace }}"
           annotations: "{{ item.annotations | default('{}') | indent(width=4, first=True) }}"
           labels: "{{ item.labels | default('{}') | indent(width=4, first=True) }}"
+          type: "{{ item.type | default('') }}"
           data: "{{ item.data }}"
       loop:
         - name: argocd-tls-certs-cm
@@ -93,7 +83,7 @@
         _template:
           name: step-ca
           namespace: step-ca
-          config: |2
+          spec: |2
               entryPoints:
                 - websecure
               routes:

ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/git.yml

@@ -32,7 +32,7 @@
         _template:
           name: gitea-ssh
           namespace: gitea
-          config: |2
+          spec: |2
               entryPoints:
                 - ssh
               routes:
@@ -110,8 +110,8 @@
             - organization: mc
               body:
                 name: GitOps.ClusterAPI
-                # auto_init: true
-                # default_branch: main
+                auto_init: true
+                default_branch: main
                 description: ClusterAPI manifests
             - organization: mc
               body:
@@ -122,15 +122,15 @@
             - organization: wl
               body:
                 name: GitOps.Config
-                # auto_init: true
-                # default_branch: main
+                auto_init: true
+                default_branch: main
                 description: GitOps manifests
             - organization: wl
               body:
-                name: GitOps.HelmCharts
-                # auto_init: true
-                # default_branch: main
-                description: Helm charts
+                name: ClusterAccess.Store
+                auto_init: true
+                default_branch: main
+                description: Kubeconfig files
           loop_control:
             label: "{{ item.organization ~ '/' ~ item.body.name }}"
 

ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/ingress.yml

@@ -27,7 +27,7 @@
     _template:
       name: traefik-dashboard
       namespace: kube-system
-      config: |2
+      spec: |2
           entryPoints:
           - web
           - websecure

ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/init.yml

@@ -12,6 +12,15 @@
     - registry
     - storage
 
+- name: Create step-ca config dictionary
+  ansible.builtin.set_fact:
+    stepconfig: "{{ { 'path': ansible_env.HOME ~ '/.step/config/values.yaml' } }}"
+
+- name: Create step-ca target folder
+  ansible.builtin.file:
+    path: "{{ stepconfig.path | dirname }}"
+    state: directory
+
 - name: Initialize tempfile
   ansible.builtin.tempfile:
     state: file
@@ -36,8 +45,8 @@
         --address=:9000 \
         --provisioner=admin \
         --acme \
-        --password-file={{ stepca_password.path }}
-  register: stepca_values
+        --password-file={{ stepca_password.path }} | tee {{ stepconfig.path }}
+    creates: "{{ stepconfig.path }}"
 
 - name: Cleanup tempfile
   ansible.builtin.file:
@@ -48,12 +57,20 @@
 - name: Store root CA certificate
   ansible.builtin.copy:
     dest: /usr/local/share/ca-certificates/root_ca.crt
-    content: "{{ (stepca_values.stdout | from_yaml).inject.certificates.root_ca }}"
+    content: "{{ (lookup('ansible.builtin.file', stepconfig.path) | from_yaml).inject.certificates.root_ca }}"
 
 - name: Update certificate truststore
   ansible.builtin.command:
     cmd: update-ca-certificates
 
+- name: Extract container images (for idempotency purposes)
+  ansible.builtin.unarchive:
+    src: /opt/metacluster/container-images/image-tarballs.tgz
+    dest: /opt/metacluster/container-images
+    remote_src: no
+  when:
+    - lookup('ansible.builtin.fileglob', '/opt/metacluster/container-images/*.tgz') is match('.*image-tarballs.tgz')
+
 - name: Get all stored fully qualified container image names
   ansible.builtin.shell:
     cmd: >-

ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/k3s.yml

@@ -42,19 +42,30 @@
   retries: "{{ playbook.retries }}"
   delay: "{{ (storage_benchmark | int) * (playbook.delay.medium | int) }}"
 
-- name: Install kubectl tab-completion
+- name: Install tab-completion
   ansible.builtin.shell:
-    cmd: kubectl completion bash | tee /etc/bash_completion.d/kubectl
+    cmd: |-
+      {{ item }} completion bash > /etc/bash_completion.d/{{ item }}
+    creates: /etc/bash_completion.d/{{ item }}
+  loop:
+    - kubectl
+    - helm
+    - step
 
-- name: Initialize tempfile
-  ansible.builtin.tempfile:
-    state: file
-  register: kubeconfig
+- name: Create kubeconfig dictionary
+  ansible.builtin.set_fact:
+    kubeconfig: "{{ { 'path': ansible_env.HOME ~ '/.kube/config' } }}"
+
+- name: Create kubeconfig target folder
+  ansible.builtin.file:
+    path: "{{ kubeconfig.path | dirname }}"
+    state: directory
 
 - name: Retrieve kubeconfig
   ansible.builtin.command:
     cmd: kubectl config view --raw
   register: kubectl_config
+  no_log: true
 
 - name: Store kubeconfig in tempfile
   ansible.builtin.copy:

ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/main.yml

@@ -1,10 +1,12 @@
 - import_tasks: init.yml
 - import_tasks: k3s.yml
 - import_tasks: assets.yml
-- import_tasks: kube-vip.yml
+- import_tasks: virtualip.yml
+- import_tasks: metadata.yml
 - import_tasks: storage.yml
 - import_tasks: ingress.yml
 - import_tasks: certauthority.yml
 - import_tasks: registry.yml
 - import_tasks: git.yml
 - import_tasks: gitops.yml
+- import_tasks: authentication.yml

ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/metadata.yml

@@ -0,0 +1,57 @@
+- block:
+    - name: Aggregate manifest-component versions into dictionary
+      ansible.builtin.set_fact:
+        manifest_versions: "{{ manifest_versions | default([]) + [ item | combine( {'type': 'manifest', 'id': index } ) ] }}"
+      loop:
+        - name: cluster-api
+          versions:
+            management:
+              base: "{{ components.clusterapi.management.version.base }}"
+              cert_manager: "{{ components.clusterapi.management.version.cert_manager }}"
+              infrastructure_vsphere: "{{ components.clusterapi.management.version.infrastructure_vsphere }}"
+              ipam_incluster: "{{ components.clusterapi.management.version.ipam_incluster }}"
+              cpi_vsphere: "{{ components.clusterapi.management.version.cpi_vsphere }}"
+            workload:
+              calico: "{{ components.clusterapi.workload.version.calico }}"
+              k8s: "{{ components.clusterapi.workload.version.k8s }}"
+        - name: kube-vip
+          version: "{{ components.kubevip.version }}"
+      loop_control:
+        label: "{{ item.name }}"
+        index_var: index
+
+    - name: Install json-server chart
+      kubernetes.core.helm:
+        name: json-server
+        chart_ref: /opt/metacluster/helm-charts/json-server
+        release_namespace: json-server
+        create_namespace: true
+        wait: false
+        kubeconfig: "{{ kubeconfig.path }}"
+        values: |
+          {{
+            components['json-server'].chart_values |
+            combine(
+              { 'jsonServer': { 'seedData': { 'configInline': (
+                { 'appliance': { "version": appliance.version }, 'components': manifest_versions, 'healthz': { 'status': 'running' } }
+              ) | to_json } } }
+            )
+          }}
+
+    - name: Ensure json-server API availability
+      ansible.builtin.uri:
+        url: https://version.{{ vapp['metacluster.fqdn'] }}/healthz
+        method: GET
+        # This mock REST API -ironically- does not support json encoded body argument
+        body_format: raw
+      register: api_readycheck
+      until:
+        - api_readycheck.json.status is defined
+        - api_readycheck.json.status == 'running'
+      retries: "{{ playbook.retries }}"
+      delay: "{{ (storage_benchmark | int) * (playbook.delay.long | int) }}"
+
+  module_defaults:
+    ansible.builtin.uri:
+      validate_certs: no
+      status_code: [200, 201]

ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/authentication.yml

@@ -0,0 +1,36 @@
+- name: Initialize tempfolder
+  ansible.builtin.tempfile:
+    state: directory
+  register: pinniped_kubeconfig
+
+- name: Pull existing repository
+  ansible.builtin.git:
+    repo: https://git.{{ vapp['metacluster.fqdn'] }}/wl/ClusterAccess.Store.git
+    dest: "{{ pinniped_kubeconfig.path }}"
+    version: main
+
+- name: Generate kubeconfig
+  ansible.builtin.shell:
+    cmd: pinniped get kubeconfig --kubeconfig {{ capi_kubeconfig.path }}
+  register: pinniped_config
+
+- name: Store kubeconfig in tempfile
+  ansible.builtin.copy:
+    dest: "{{ pinniped_kubeconfig.path }}"
+    content: "{{ pinniped_config.stdout }}"
+    mode: 0600
+  no_log: true
+
+- name: Push git repository
+  lvrfrc87.git_acp.git_acp:
+    path: "{{ pinniped_kubeconfig.path }}"
+    branch: main
+    comment: "Upload kubeconfig files"
+    add:
+      - .
+    url: https://administrator:{{ vapp['metacluster.password'] | urlencode }}@git.{{ vapp['metacluster.fqdn'] }}/wl/ClusterAccess.Store.git
+  environment:
+    GIT_AUTHOR_NAME: administrator
+    GIT_AUTHOR_EMAIL: administrator@{{ vapp['metacluster.fqdn'] }}
+    GIT_COMMITTER_NAME: administrator
+    GIT_COMMITTER_EMAIL: administrator@{{ vapp['metacluster.fqdn'] }}

ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/clusterapi.yml

@@ -129,6 +129,12 @@
     state: directory
   register: capi_clustermanifest
 
+- name: Pull existing repository
+  ansible.builtin.git:
+    repo: https://git.{{ vapp['metacluster.fqdn'] }}/mc/GitOps.ClusterAPI.git
+    dest: "{{ capi_clustermanifest.path }}"
+    version: main
+
 - name: Save workload cluster manifest
   ansible.builtin.copy:
     dest: "{{ capi_clustermanifest.path }}/new-cluster.yaml"
@@ -173,24 +179,46 @@
           prefix: "{{ vapp['guestinfo.prefixlength'] }}"
           gateway: "{{ vapp['guestinfo.gateway'] }}"
 
-- name: Initialize/Push git repository
-  ansible.builtin.shell:
-    cmd: |
-      git init
-      git config --global user.email "administrator@{{ vapp['metacluster.fqdn'] }}"
-      git config --global user.name "administrator"
-      git checkout -b main
-      git add ./manifests
-      git commit -m "Upload manifests"
-      git remote add origin https://git.{{ vapp['metacluster.fqdn'] }}/mc/GitOps.ClusterAPI.git
-      git push https://administrator:{{ vapp['metacluster.password'] | urlencode }}@git.{{ vapp['metacluster.fqdn'] }}/mc/GitOps.ClusterAPI.git --all
-    chdir: "{{ capi_clustermanifest.path }}"
-
-- name: Cleanup tempfolder
+- name: Remove working files
   ansible.builtin.file:
-    path: "{{ capi_clustermanifest.path }}"
+    path: "{{ item }}"
     state: absent
-  when: capi_clustermanifest.path is defined
+  loop: "{{ query('ansible.builtin.fileglob', capi_clustermanifest.path ~ '/*.yaml') }}"
+  loop_control:
+    label: "{{ item | basename }}"
+
+- name: Push git repository
+  lvrfrc87.git_acp.git_acp:
+    path: "{{ capi_clustermanifest.path }}"
+    branch: main
+    comment: "Upload manifests"
+    add:
+      - ./manifests
+    url: https://administrator:{{ vapp['metacluster.password'] | urlencode }}@git.{{ vapp['metacluster.fqdn'] }}/mc/GitOps.ClusterAPI.git
+  environment:
+    GIT_AUTHOR_NAME: administrator
+    GIT_AUTHOR_EMAIL: administrator@{{ vapp['metacluster.fqdn'] }}
+    GIT_COMMITTER_NAME: administrator
+    GIT_COMMITTER_EMAIL: administrator@{{ vapp['metacluster.fqdn'] }}
+
+# - name: Initialize/Push git repository
+#   ansible.builtin.shell:
+#     cmd: |
+#       git init
+#       git config --global user.email "administrator@{{ vapp['metacluster.fqdn'] }}"
+#       git config --global user.name "administrator"
+#       git checkout -b main
+#       git add ./manifests
+#       git commit -m "Upload manifests"
+#       git remote add origin https://git.{{ vapp['metacluster.fqdn'] }}/mc/GitOps.ClusterAPI.git
+#       git push https://administrator:{{ vapp['metacluster.password'] | urlencode }}@git.{{ vapp['metacluster.fqdn'] }}/mc/GitOps.ClusterAPI.git --all
+#     chdir: "{{ capi_clustermanifest.path }}"
+
+# - name: Cleanup tempfolder
+#   ansible.builtin.file:
+#     path: "{{ capi_clustermanifest.path }}"
+#     state: absent
+#   when: capi_clustermanifest.path is defined
 
 - name: Configure Cluster API repository
   ansible.builtin.template:
@@ -277,7 +305,12 @@
 # TODO: move to git repo
 - name: Apply cni plugin manifest
   kubernetes.core.k8s:
-    src: /opt/metacluster/cluster-api/cni-calico/{{ components.clusterapi.workload.version.calico }}/calico.yaml
+    definition: |
+      {{
+        lookup('ansible.builtin.file', '/opt/metacluster/cluster-api/cni-calico/' ~ components.clusterapi.workload.version.calico ~ '/calico.yaml') |
+          regex_replace('# - name: CALICO_IPV4POOL_CIDR', '- name: CALICO_IPV4POOL_CIDR') |
+          regex_replace('#   value: "192.168.0.0/16"',    '  value: "172.30.0.0/16"')
+      }}
     state: present
     wait: true
     kubeconfig: "{{ capi_kubeconfig.path }}"

ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/gitops.yml

@@ -5,6 +5,20 @@
     recurse: false
   register: helm_charts
 
+- name: Pull existing repository
+  ansible.builtin.git:
+    repo: https://git.{{ vapp['metacluster.fqdn'] }}/wl/GitOps.Config.git
+    dest: /opt/workloadcluster/git-repositories/gitops
+    version: main
+
+- name: Create folder structure within new git-repository
+  ansible.builtin.file:
+    path: "{{ item }}"
+    state: directory
+  loop:
+    - /opt/workloadcluster/git-repositories/gitops/charts
+    - /opt/workloadcluster/git-repositories/gitops/values
+
 - name: Create hard-links to populate new git-repository
   ansible.builtin.shell:
     cmd: >-
@@ -13,6 +27,18 @@
   loop_control:
     label: "{{ item.path | basename }}"
 
+- name: Write custom manifests to respective chart templates store
+  ansible.builtin.template:
+    src: "{{ src }}"
+    dest: /opt/workloadcluster/git-repositories/gitops/charts/{{ manifest.value.namespace }}/{{ manifest.key }}/templates/{{ (src | split('.'))[0] ~ '-' ~ _template.name ~ '.yaml' }}
+  vars:
+    manifest: "{{ item.0 }}"
+    src: "{{ item.1.src }}"
+    _template: "{{ item.1._template }}"
+  loop: "{{ query('ansible.builtin.subelements', query('ansible.builtin.dict', downstream_components), 'value.extra_manifests') }}"
+  loop_control:
+    label: "{{ (src | split('.'))[0] ~ '-' ~ _template.name }}"
+
 - name: Create subfolders
   ansible.builtin.file:
     path: /opt/workloadcluster/git-repositories/gitops/values/{{ item.key }}
@@ -29,18 +55,19 @@
   loop_control:
     label: "{{ item.key }}"
 
-- name: Initialize/Push git repository
-  ansible.builtin.shell:
-    cmd: |
-      git init
-      git config --global user.email "administrator@{{ vapp['metacluster.fqdn'] }}"
-      git config --global user.name "administrator"
-      git checkout -b main
-      git add .
-      git commit -m "Upload charts"
-      git remote add origin https://git.{{ vapp['metacluster.fqdn'] }}/wl/GitOps.Config.git
-      git push https://administrator:{{ vapp['metacluster.password'] | urlencode }}@git.{{ vapp['metacluster.fqdn'] }}/wl/GitOps.Config.git --all
-    chdir: /opt/workloadcluster/git-repositories/gitops
+- name: Push git repository
+  lvrfrc87.git_acp.git_acp:
+    path: /opt/workloadcluster/git-repositories/gitops
+    branch: main
+    comment: "Upload charts"
+    add:
+      - .
+    url: https://administrator:{{ vapp['metacluster.password'] | urlencode }}@git.{{ vapp['metacluster.fqdn'] }}/wl/GitOps.Config.git
+  environment:
+    GIT_AUTHOR_NAME: administrator
+    GIT_AUTHOR_EMAIL: administrator@{{ vapp['metacluster.fqdn'] }}
+    GIT_COMMITTER_NAME: administrator
+    GIT_COMMITTER_EMAIL: administrator@{{ vapp['metacluster.fqdn'] }}
 
 - name: Retrieve workload-cluster kubeconfig
   kubernetes.core.k8s_info:

ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/main.yml

@@ -6,6 +6,7 @@
 
     - import_tasks: clusterapi.yml
     - import_tasks: gitops.yml
+    - import_tasks: authentication.yml
 
   when:
     - vapp['deployment.type'] != 'core'

ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/federationdomain.j2

@@ -0,0 +1,7 @@
+apiVersion: config.supervisor.pinniped.dev/v1alpha1
+kind: FederationDomain
+metadata:
+  name: {{ _template.name }}
+  namespace: {{ _template.namespace }}
+spec:
+{{ _template.spec }}

ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/ingressroute.j2

@@ -4,4 +4,4 @@ metadata:
   name: {{ _template.name }}
   namespace: {{ _template.namespace }}
 spec:
-{{ _template.config }}
+{{ _template.spec }}

ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/ingressroutetcp.j2

@@ -4,4 +4,4 @@ metadata:
   name: {{ _template.name }}
   namespace: {{ _template.namespace }}
 spec:
-{{ _template.config }}
+{{ _template.spec }}

ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/jwtauthenticator.j2

@@ -0,0 +1,6 @@
+apiVersion: authentication.concierge.pinniped.dev/v1alpha1
+kind: JWTAuthenticator
+metadata:
+  name: {{ _template.name }}
+spec:
+{{ _template.spec }}

ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/kustomization.cluster-template.j2

@@ -3,8 +3,8 @@ kind: Kustomization
 resources:
 - cluster-template.yaml
 
-patchesStrategicMerge:
-  - |-
+patches:
+- patch: |-
     apiVersion: v1
     kind: Secret
     metadata:
@@ -32,7 +32,7 @@ patchesStrategicMerge:
             [Network]
             public-network = "${VSPHERE_NETWORK}"
         type: Opaque
-  - |-
+- patch: |-
     apiVersion: controlplane.cluster.x-k8s.io/v1beta1
     kind: KubeadmControlPlane
     metadata:
@@ -42,7 +42,7 @@ patchesStrategicMerge:
       kubeadmConfigSpec:
         clusterConfiguration:
           imageRepository: registry.{{ _template.network.fqdn }}/kubeadm
-  - |-
+- patch: |-
     apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
     kind: KubeadmConfigTemplate
     metadata:
@@ -53,7 +53,7 @@ patchesStrategicMerge:
         spec:
           clusterConfiguration:
             imageRepository: registry.{{ _template.network.fqdn }}/kubeadm
-  - |-
+- patch: |-
     apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
     kind: KubeadmConfigTemplate
     metadata:
@@ -86,7 +86,7 @@ patchesStrategicMerge:
               {{ _template.rootca | indent(width=14, first=False) | trim }}
             owner: root:root
             path: /usr/local/share/ca-certificates/root_ca.crt
-  - |-
+- patch: |-
     apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
     kind: VSphereMachineTemplate
     metadata:
@@ -105,7 +105,7 @@ patchesStrategicMerge:
               nameservers:
               - {{ _template.network.dnsserver }}
               networkName: '${VSPHERE_NETWORK}'
-  - |-
+- patch: |-
     apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
     kind: VSphereMachineTemplate
     metadata:
@@ -125,132 +125,136 @@ patchesStrategicMerge:
               - {{ _template.network.dnsserver }}
               networkName: '${VSPHERE_NETWORK}'
 
-patchesJson6902:
-  - target:
-      group: controlplane.cluster.x-k8s.io
-      version: v1beta1
-      kind: KubeadmControlPlane
-      name: .*
-    patch: |-
-      - op: add
-        path: /spec/kubeadmConfigSpec/files/-
-        value:
-          content: |
-              [plugins."io.containerd.grpc.v1.cri".registry]
-                config_path = "/etc/containerd/certs.d"
-          append: true
-          path: /etc/containerd/config.toml
+- target:
+    group: controlplane.cluster.x-k8s.io
+    version: v1beta1
+    kind: KubeadmControlPlane
+    name: .*
+  patch: |-
+    - op: add
+      path: /spec/kubeadmConfigSpec/files/-
+      value:
+        content: |
+            [plugins."io.containerd.grpc.v1.cri".registry]
+              config_path = "/etc/containerd/certs.d"
+        append: true
+        path: /etc/containerd/config.toml
 {% for registry in _template.registries %}
-      - op: add
-        path: /spec/kubeadmConfigSpec/files/-
-        value:
-          content: |
-            server = "https://{{ registry }}"
+    - op: add
+      path: /spec/kubeadmConfigSpec/files/-
+      value:
+        content: |
+          server = "https://{{ registry }}"
 
-            [host."https://registry.{{ _template.network.fqdn }}/v2/library/{{ registry }}"]
-              capabilities = ["pull", "resolve"]
-              override_path = true
-          owner: root:root
-          path: /etc/containerd/certs.d/{{ registry }}/hosts.toml
+          [host."https://registry.{{ _template.network.fqdn }}/v2/library/{{ registry }}"]
+            capabilities = ["pull", "resolve"]
+            override_path = true
+        owner: root:root
+        path: /etc/containerd/certs.d/{{ registry }}/hosts.toml
 {% endfor %}
-      - op: add
-        path: /spec/kubeadmConfigSpec/files/-
-        value:
-          content: |
-            network: {config: disabled}
-          owner: root:root
-          path: /etc/cloud/cloud.cfg.d/99-disable-network-config.cfg
-      - op: add
-        path: /spec/kubeadmConfigSpec/files/-
-        value:
-          content: |
-            {{ _template.rootca | indent(width=12, first=False) | trim }}
-          owner: root:root
-          path: /usr/local/share/ca-certificates/root_ca.crt
-  - target:
-      group: bootstrap.cluster.x-k8s.io
-      version: v1beta1
-      kind: KubeadmConfigTemplate
-      name: .*
-    patch: |-
+    - op: add
+      path: /spec/kubeadmConfigSpec/files/-
+      value:
+        content: |
+          network: {config: disabled}
+        owner: root:root
+        path: /etc/cloud/cloud.cfg.d/99-disable-network-config.cfg
+    - op: add
+      path: /spec/kubeadmConfigSpec/files/-
+      value:
+        content: |
+          {{ _template.rootca | indent(width=10, first=False) | trim }}
+        owner: root:root
+        path: /usr/local/share/ca-certificates/root_ca.crt
+- target:
+    group: bootstrap.cluster.x-k8s.io
+    version: v1beta1
+    kind: KubeadmConfigTemplate
+    name: .*
+  patch: |-
 {% for cmd in _template.runcmds %}
-      - op: add
-        path: /spec/template/spec/preKubeadmCommands/-
-        value: {{ cmd }}
+    - op: add
+      path: /spec/template/spec/preKubeadmCommands/-
+      value: {{ cmd }}
 {% endfor %}
-  - target:
-      group: controlplane.cluster.x-k8s.io
-      version: v1beta1
-      kind: KubeadmControlPlane
-      name: .*
-    patch: |-
+- target:
+    group: controlplane.cluster.x-k8s.io
+    version: v1beta1
+    kind: KubeadmControlPlane
+    name: .*
+  patch: |-
 {% for cmd in _template.runcmds %}
-      - op: add
-        path: /spec/kubeadmConfigSpec/preKubeadmCommands/-
-        value: {{ cmd }}
+    - op: add
+      path: /spec/kubeadmConfigSpec/preKubeadmCommands/-
+      value: {{ cmd }}
 {% endfor %}
 
-  - target:
-      group: infrastructure.cluster.x-k8s.io
-      version: v1beta1
-      kind: VSphereMachineTemplate
-      name: \${CLUSTER_NAME}
-    patch: |-
-      - op: replace
-        path: /metadata/name
-        value: ${CLUSTER_NAME}-master
-  - target:
-      group: controlplane.cluster.x-k8s.io
-      version: v1beta1
-      kind: KubeadmControlPlane
-      name: \${CLUSTER_NAME}
-    patch: |-
-      - op: replace
-        path: /metadata/name
-        value: ${CLUSTER_NAME}-master
-      - op: replace
-        path: /spec/machineTemplate/infrastructureRef/name
-        value: ${CLUSTER_NAME}-master
-  - target:
-      group: cluster.x-k8s.io
-      version: v1beta1
-      kind: Cluster
-      name: \${CLUSTER_NAME}
-    patch: |-
-      - op: replace
-        path: /spec/controlPlaneRef/name
-        value: ${CLUSTER_NAME}-master
+- target:
+    group: infrastructure.cluster.x-k8s.io
+    version: v1beta1
+    kind: VSphereMachineTemplate
+    name: \${CLUSTER_NAME}
+  patch: |-
+    - op: replace
+      path: /metadata/name
+      value: ${CLUSTER_NAME}-master
+- target:
+    group: controlplane.cluster.x-k8s.io
+    version: v1beta1
+    kind: KubeadmControlPlane
+    name: \${CLUSTER_NAME}
+  patch: |-
+    - op: replace
+      path: /metadata/name
+      value: ${CLUSTER_NAME}-master
+    - op: replace
+      path: /spec/machineTemplate/infrastructureRef/name
+      value: ${CLUSTER_NAME}-master
+- target:
+    group: cluster.x-k8s.io
+    version: v1beta1
+    kind: Cluster
+    name: \${CLUSTER_NAME}
+  patch: |-
+    - op: replace
+      path: /spec/clusterNetwork/pods
+      value:
+        cidrBlocks:
+          - 172.30.0.0/16
+    - op: replace
+      path: /spec/controlPlaneRef/name
+      value: ${CLUSTER_NAME}-master
 
-  - target:
-      group: infrastructure.cluster.x-k8s.io
-      version: v1beta1
-      kind: VSphereMachineTemplate
-      name: \${CLUSTER_NAME}-worker
-    patch: |-
-      - op: replace
-        path: /spec/template/spec/numCPUs
-        value: {{ _template.nodesize.cpu }}
-      - op: replace
-        path: /spec/template/spec/memoryMiB
-        value: {{ _template.nodesize.memory }}
-  - target:
-      group: cluster.x-k8s.io
-      version: v1beta1
-      kind: MachineDeployment
-      name: \${CLUSTER_NAME}-md-0
-    patch: |-
-      - op: replace
-        path: /metadata/name
-        value: ${CLUSTER_NAME}-worker
-      - op: replace
-        path: /spec/template/spec/bootstrap/configRef/name
-        value: ${CLUSTER_NAME}-worker
-  - target:
-      group: bootstrap.cluster.x-k8s.io
-      version: v1beta1
-      kind: KubeadmConfigTemplate
-      name: \${CLUSTER_NAME}-md-0
-    patch: |-
-      - op: replace
-        path: /metadata/name
-        value: ${CLUSTER_NAME}-worker
+- target:
+    group: infrastructure.cluster.x-k8s.io
+    version: v1beta1
+    kind: VSphereMachineTemplate
+    name: \${CLUSTER_NAME}-worker
+  patch: |-
+    - op: replace
+      path: /spec/template/spec/numCPUs
+      value: {{ _template.nodesize.cpu }}
+    - op: replace
+      path: /spec/template/spec/memoryMiB
+      value: {{ _template.nodesize.memory }}
+- target:
+    group: cluster.x-k8s.io
+    version: v1beta1
+    kind: MachineDeployment
+    name: \${CLUSTER_NAME}-md-0
+  patch: |-
+    - op: replace
+      path: /metadata/name
+      value: ${CLUSTER_NAME}-worker
+    - op: replace
+      path: /spec/template/spec/bootstrap/configRef/name
+      value: ${CLUSTER_NAME}-worker
+- target:
+    group: bootstrap.cluster.x-k8s.io
+    version: v1beta1
+    kind: KubeadmConfigTemplate
+    name: \${CLUSTER_NAME}-md-0
+  patch: |-
+    - op: replace
+      path: /metadata/name
+      value: ${CLUSTER_NAME}-worker

ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/kustomization.nodepool.j2

@@ -5,8 +5,8 @@ resources:
 - manifests/machinedeployment-{{ _template.cluster.name }}-worker.yaml
 - manifests/vspheremachinetemplate-{{ _template.cluster.name }}-worker.yaml
 
-patchesStrategicMerge:
-  - |-
+patches:
+- patch: |-
     apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
     kind: KubeadmConfigTemplate
     metadata:
@@ -31,7 +31,7 @@ patchesStrategicMerge:
           mounts:
           - - LABEL=blockstorage
             - /mnt/blockstorage
-  - |-
+- patch: |-
     apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
     kind: VSphereMachineTemplate
     metadata:
@@ -43,42 +43,41 @@ patchesStrategicMerge:
           additionalDisksGiB:
           - {{ _template.nodepool.additionaldisk }}
 
-patchesJson6902:
-  - target:
-      group: bootstrap.cluster.x-k8s.io
-      version: v1beta1
-      kind: KubeadmConfigTemplate
-      name: {{ _template.cluster.name }}-worker
-    patch: |-
-      - op: replace
-        path: /metadata/name
-        value: {{ _template.cluster.name }}-worker-storage
+- target:
+    group: bootstrap.cluster.x-k8s.io
+    version: v1beta1
+    kind: KubeadmConfigTemplate
+    name: {{ _template.cluster.name }}-worker
+  patch: |-
+    - op: replace
+      path: /metadata/name
+      value: {{ _template.cluster.name }}-worker-storage
 
-  - target:
-      group: cluster.x-k8s.io
-      version: v1beta1
-      kind: MachineDeployment
-      name: {{ _template.cluster.name }}-worker
-    patch: |-
-      - op: replace
-        path: /metadata/name
-        value: {{ _template.cluster.name }}-worker-storage
-      - op: replace
-        path: /spec/template/spec/bootstrap/configRef/name
-        value: {{ _template.cluster.name }}-worker-storage
-      - op: replace
-        path: /spec/template/spec/infrastructureRef/name
-        value: {{ _template.cluster.name }}-worker-storage
-      - op: replace
-        path: /spec/replicas
-        value: {{ _template.nodepool.size }}
+- target:
+    group: cluster.x-k8s.io
+    version: v1beta1
+    kind: MachineDeployment
+    name: {{ _template.cluster.name }}-worker
+  patch: |-
+    - op: replace
+      path: /metadata/name
+      value: {{ _template.cluster.name }}-worker-storage
+    - op: replace
+      path: /spec/template/spec/bootstrap/configRef/name
+      value: {{ _template.cluster.name }}-worker-storage
+    - op: replace
+      path: /spec/template/spec/infrastructureRef/name
+      value: {{ _template.cluster.name }}-worker-storage
+    - op: replace
+      path: /spec/replicas
+      value: {{ _template.nodepool.size }}
 
-  - target:
-      group: infrastructure.cluster.x-k8s.io
-      version: v1beta1
-      kind: VSphereMachineTemplate
-      name: {{ _template.cluster.name }}-worker
-    patch: |-
-      - op: replace
-        path: /metadata/name
-        value: {{ _template.cluster.name }}-worker-storage
+- target:
+    group: infrastructure.cluster.x-k8s.io
+    version: v1beta1
+    kind: VSphereMachineTemplate
+    name: {{ _template.cluster.name }}-worker
+  patch: |-
+    - op: replace
+      path: /metadata/name
+      value: {{ _template.cluster.name }}-worker-storage

ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/oidcidentityprovider.j2

@@ -0,0 +1,7 @@
+apiVersion: idp.supervisor.pinniped.dev/v1alpha1
+kind: OIDCIdentityProvider
+metadata:
+  name: {{ _template.name }}
+  namespace: {{ _template.namespace }}
+spec:
+{{ _template.spec }}

ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/secret.j2

@@ -3,6 +3,7 @@ kind: Secret
 metadata:
   name: {{ _template.name }}
   namespace: {{ _template.namespace }}
+type: {{ _template.type }}
 data:
 {% for kv_pair in _template.data %}
   "{{ kv_pair.key }}": {{ kv_pair.value }}

ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/serverstransport.j2

@@ -0,0 +1,7 @@
+apiVersion: traefik.containo.us/v1alpha1
+kind: ServersTransport
+metadata:
+  name: {{ _template.name }}
+  namespace: {{ _template.namespace }}
+spec:
+{{ _template.spec }}

ansible/roles/firstboot/files/ansible_payload/common/roles/cleanup/tasks/main.yml

@@ -1,12 +1,6 @@
 - import_tasks: service.yml
 - import_tasks: cron.yml
 
-- name: Cleanup tempfile
-  ansible.builtin.file:
-    path: "{{ kubeconfig.path }}"
-    state: absent
-  when: kubeconfig.path is defined
-
 # - name: Reboot host
 #   ansible.builtin.shell:
 #     cmd: systemctl reboot

ansible/vars/metacluster.yml

@@ -1,7 +1,7 @@
 platform:
 
   k3s:
-    version: v1.25.9+k3s1
+    version: v1.27.1+k3s1
 
   packaged_components:
     - name: traefik
@@ -33,12 +33,10 @@ platform:
   helm_repositories:
     - name: argo
       url: https://argoproj.github.io/argo-helm
-    - name: authentik
-      url: https://charts.goauthentik.io
-    # - name: codecentric
-    #   url: https://codecentric.github.io/helm-charts
-    # - name: dex
-    #   url: https://charts.dexidp.io
+    - name: bitnami
+      url: https://charts.bitnami.com/bitnami
+    - name: dexidp
+      url: https://charts.dexidp.io
     - name: gitea-charts
       url: https://dl.gitea.io/charts/
     - name: harbor
@@ -51,6 +49,8 @@ platform:
       url: https://prometheus-community.github.io/helm-charts
     - name: smallstep
       url: https://smallstep.github.io/helm-charts/
+    - name: spamasaurus
+      url: https://code.spamasaurus.com/api/packages/djpbessems/helm
 
 components:
 
@@ -71,32 +71,6 @@ components:
             hosts:
               - gitops.{{ vapp['metacluster.fqdn'] }}
 
-  authentik:
-    helm:
-      version: 2023.3.1
-      chart: authentik/authentik
-      parse_logic: helm template . --set postgresql.enabled=true,redis.enabled=true | yq --no-doc eval '.. | .image? | select(.)' | sort -u | awk '!/ /'
-      chart_values: !unsafe |
-        authentik:
-          avatars: none
-          secret_key: "{{ lookup('ansible.builtin.password', '/dev/null length=64 chars=ascii_lowercase,digits seed=' ~ vapp['guestinfo.hostname']) }}"
-          postgresql:
-            password: "{{ lookup('ansible.builtin.password', '/dev/null length=32 chars=ascii_lowercase,digits seed=' ~ vapp['guestinfo.hostname']) }}"
-        env:
-          AUTHENTIK_BOOTSTRAP_PASSWORD: "{{ vapp['metacluster.password'] }}"
-        ingress:
-          enabled: true
-          hosts:
-            - host: auth.{{ vapp['metacluster.fqdn'] }}
-              paths:
-                - path: "/"
-                  pathType: Prefix
-        postgresql:
-          enabled: true
-          postgresqlPassword: "{{ lookup('ansible.builtin.password', '/dev/null length=32 chars=ascii_lowercase,digits seed=' ~ vapp['guestinfo.hostname']) }}"
-        redis:
-          enabled: true
-
   cert-manager:
     helm:
       version: 1.11.0
@@ -115,61 +89,45 @@ components:
         infrastructure_vsphere: v1.6.0
         ipam_incluster: v0.1.0-alpha.2
         # Refer to `https://console.cloud.google.com/gcr/images/cloud-provider-vsphere/GLOBAL/cpi/release/manager` for available tags
-        cpi_vsphere: v1.25.2
+        cpi_vsphere: v1.26.2
     workload:
       version:
         calico: v3.25.0
-        k8s: v1.25.9
+        k8s: v1.27.1
       node_template:
-        url: https://{{ repo_username }}:{{ repo_password }}@sn.itch.fyi/Repository/rel/ubuntu-2204-kube-v1.25.9.ova
+        url: https://{{ repo_username }}:{{ repo_password }}@sn.itch.fyi/Repository/rel/ubuntu-2204-kube-v1.27.1.ova
 
-  # dex:
-  #   helm:
-  #     version: 0.13.0 # (= Dex 2.35.3)
-  #     chart: dex/dex
-  #     parse_logic: helm template . | yq --no-doc eval '.. | .image? | select(.)' | sort -u | awk '!/ /'
-  #     chart_values: !unsafe |
-  #       config:
-  #         connectors:
-  #           - type: ldap
-  #             id: ldap
-  #             name: "LDAP"
-  #             config:
-  #               host: "{{ vapp['ldap.fqdn'] }}:636"
-  #               insecureNoSSL: false
-  #               insecureSkipVerify: true
-  #               bindDN: "{{ vapp['ldap.dn'] }}"
-  #               bindPW: "{{ vapp['ldap.password'] }}"
-
-  #               usernamePrompt: "Username"
-  #               userSearch:
-  #                 baseDN: OU=Administrators,OU=Useraccounts,DC=bessems,DC=eu
-  #                 filter: "(objectClass=person)"
-  #                 username: userPrincipalName
-  #                 idAttr: DN
-  #                 emailAttr: userPrincipalName
-  #                 nameAttr: cn
-
-  #               groupSearch:
-  #                 baseDN: OU=Roles,OU=Groups,DC=bessems,DC=eu
-  #                 filter: "(objectClass=group)"
-  #                 userMatchers:
-  #                 - userAttr: DN
-  #                   groupAttr: member
-  #                 nameAttr: cn
-  #         enablePasswordDB: true
-  #         issuer: https://oidc.{{ vapp['metacluster.fqdn'] }}
-  #         storage:
-  #           type: kubernetes
-  #           config:
-  #             inCluster: true
-  #       ingress:
-  #         enabled: true
-  #         hosts:
-  #           - host: oidc.{{ vapp['metacluster.fqdn'] }}
-  #             paths:
-  #               - path: /
-  #                 pathType: Prefix
+  dex:
+    helm:
+      version: 0.15.3  # (= Dex 2.37.0)
+      chart: dexidp/dex
+      parse_logic: helm template . | yq --no-doc eval '.. | .image? | select(.)' | sort -u | awk '!/ /'
+      chart_values: !unsafe |
+        config:
+          issuer: https://idps.{{ vapp['metacluster.fqdn'] }}
+          storage:
+            type: kubernetes
+            config:
+              inCluster: true
+          staticClients:
+          - id: pinniped-supervisor
+            secret: "{{ lookup('ansible.builtin.password', '/dev/null length=64 chars=ascii_lowercase,digits seed=' ~ vapp['metacluster.fqdn']) }}"
+            name: Pinniped Supervisor client
+            redirectURIs:
+            - https://auth.{{ vapp['metacluster.fqdn'] }}/sso/callback
+          enablePasswordDB: true
+          staticPasswords:
+          - email: user@{{ vapp['metacluster.fqdn'] }}
+            hash: "{{ vapp['metacluster.password'] | password_hash('bcrypt') }}"
+            username: user
+            userID: "{{ lookup('ansible.builtin.password', '/dev/null length=64 chars=ascii_lowercase,digits seed=' ~ vapp['metacluster.fqdn']) | to_uuid }}"
+        ingress:
+          enabled: true
+          hosts:
+            - host: idps.{{ vapp['metacluster.fqdn'] }}
+              paths:
+                - path: /
+                  pathType: Prefix
 
   gitea:
     helm:
@@ -225,37 +183,24 @@ components:
             registry:
               size: 25Gi
 
-  # keycloakx:
-  #   helm:
-  #     version: 2.1.1  # (= Keycloak 20.0.3)
-  #     chart: codecentric/keycloakx
-  #     parse_logic: helm template . | yq --no-doc eval '.. | .image? | select(.)' | sort -u | awk '!/ /'
-  #     chart_values: !unsafe |
-  #       command:
-  #         - "/opt/keycloak/bin/kc.sh"
-  #         - "start"
-  #         - "--http-enabled=true"
-  #         - "--http-port=8080"
-  #         - "--hostname-strict=false"
-  #         - "--hostname-strict-https=false"
-  #       extraEnv: |
-  #         - name: KEYCLOAK_ADMIN
-  #           value: admin
-  #         - name: KEYCLOAK_ADMIN_PASSWORD
-  #           value: {{ vapp['metacluster.password'] }}
-  #         - name: KC_PROXY
-  #           value: "passthrough"
-  #         - name: JAVA_OPTS_APPEND
-  #           value: >-
-  #             -Djgroups.dns.query={% raw %}{{ include "keycloak.fullname" . }}{% endraw %}-headless
-  #       ingress:
-  #         enabled: true
-  #         rules:
-  #           - host: keycloak.{{ vapp['metacluster.fqdn'] }}
-  #             paths:
-  #               - path: /
-  #                 pathType: Prefix
-  #         tls: []
+  json-server:
+    helm:
+      version: v0.8.3
+      chart: spamasaurus/json-server
+      parse_logic: helm template . | yq --no-doc eval '.. | .image? | select(.)' | sort -u | awk '!/ /'
+      chart_values: !unsafe |
+        ingress:
+          enabled: true
+          hosts:
+            - host: version.{{ vapp['metacluster.fqdn'] }}
+              paths:
+                - path: /
+                  pathType: Prefix
+        jsonServer:
+          seedData:
+            configInline: {}
+        sidecar:
+          targetUrl: version.{{ vapp['metacluster.fqdn'] }}
 
   kube-prometheus-stack:
     helm:
@@ -288,6 +233,19 @@ components:
         persistence:
           defaultClassReplicaCount: 1
 
+  pinniped:
+    helm:
+      version: 1.2.11  # (= Pinniped v0.25.0)
+      chart: bitnami/pinniped
+      parse_logic: helm template . | yq --no-doc eval '.. | .image? | select(.)' | sort -u | awk '!/ /'
+      chart_values: !unsafe |
+        concierge:
+          enabled: false
+        supervisor:
+          service:
+            public:
+              type: ClusterIP
+
   step-certificates:
     helm:
       version: 1.23.0
@@ -316,6 +274,7 @@ dependencies:
     - community.general
     - community.vmware
     - kubernetes.core
+    - lvrfrc87.git_acp
 
   container_images:
     # This should match the image tag referenced at `platform.packaged_components[.name==traefik].config`
@@ -345,6 +304,8 @@ dependencies:
     - filename: kubectl-slice
       url: https://github.com/patrickdappollonio/kubectl-slice/releases/download/v1.2.5/kubectl-slice_linux_x86_64.tar.gz
       archive: compressed
+    - filename: pinniped
+      url: https://github.com/vmware-tanzu/pinniped/releases/download/v0.25.0/pinniped-cli-linux-amd64
     - filename: skopeo
       url: https://code.spamasaurus.com/api/packages/djpbessems/generic/skopeo/v1.12.0/skopeo_linux_amd64
     - filename: step

ansible/vars/workloadcluster.yml

@@ -1,6 +1,8 @@
 downstream:
 
   helm_repositories:
+    - name: bitnami
+      url: https://charts.bitnami.com/bitnami
     - name: longhorn
       url: https://charts.longhorn.io
     - name: sealed-secrets
@@ -18,6 +20,24 @@ downstream:
           createDefaultDiskLabeledNodes: true
           defaultDataPath: /mnt/blockstorage
 
+    pinniped:
+      version: 1.2.11  # (= Pinniped v0.25.0)
+      chart: bitnami/pinniped
+      namespace: pinniped-concierge
+      parse_logic: helm template . | yq --no-doc eval '.. | .image? | select(.)' | sort -u | awk '!/ /'
+      chart_values: !unsafe |
+        supervisor:
+          enabled: false
+      extra_manifests:
+        - src: jwtauthenticator.j2
+          _template:
+            name: metacluster-sso
+            spec: !unsafe |2
+                issuer: https://auth.{{ vapp['metacluster.fqdn'] }}/sso
+                audience: "{{ vapp['workloadcluster.name'] | lower }}"
+                tls:
+                  certificateAuthorityData: "{{ (stepca_cm_certs.resources[0].data['intermediate_ca.crt'] ~ _newline ~ stepca_cm_certs.resources[0].data['root_ca.crt']) | b64encode }}"
+
     sealed-secrets:
       version: 2.8.1  # (= Sealed Secrets v0.20.2)
       chart: sealed-secrets/sealed-secrets

packer/build.pkr.hcl

@@ -1,5 +1,14 @@
 packer {
   required_plugins {
+    vsphere = {
+      source  = "github.com/hashicorp/vsphere"
+      version = "~> 1"
+    }
+
+    ansible = {
+      source  = "github.com/hashicorp/ansible"
+      version = "~> 1"
+    }
   }
 }
 
@@ -28,6 +37,7 @@ build {
 
     extra_arguments  = [
       "--extra-vars", "appliancetype=${source.name}",
+      "--extra-vars", "applianceversion=${var.appliance_version}",
       "--extra-vars", "ansible_ssh_pass=${var.ssh_password}",
       "--extra-vars", "docker_username=${var.docker_username}",
       "--extra-vars", "docker_password=${var.docker_password}",
@@ -45,7 +55,7 @@ build {
       " -ManifestFileName '/scratch/bld_${var.vm_name}_${source.name}.mf'",
       "ovftool --acceptAllEulas --allowExtraConfig --overwrite \\",
       " '/scratch/bld_${var.vm_name}_${source.name}.ovf' \\",
-      " /output/airgapped-k8s-${var.next_release_version}+${var.k8s_version}-${source.name}.ova"
+      " /output/airgapped-k8s-${var.appliance_version}+${var.k8s_version}-${source.name}.ova"
     ]
   }
 }

packer/variables.pkr.hcl

@@ -34,5 +34,5 @@ variable "docker_password" {
     sensitive = true
 }
 
+variable "appliance_version" {}
 variable "k8s_version" {}
-variable "next_release_version" {}