feat: Add cli binary

.gitignore

@@ -0,0 +1,4 @@
+**/hv.vcenter.yaml
+**/ova.bootstrap.yaml
+**/pb.secrets.yaml
+**/pwdfile

ansible/roles/assets/tasks/main.yml

@@ -29,4 +29,3 @@
 - import_tasks: manifests.yml
 - import_tasks: kubeadm.yml
 - import_tasks: containerimages.yml
-- import_tasks: nodetemplates.yml

ansible/roles/assets/tasks/nodetemplates.yml

@@ -1,4 +0,0 @@
-- name: Download node-template image
-  ansible.builtin.uri:
-    url: "{{ components.clusterapi.workload.node_template.url }}"
-    dest: /opt/workloadcluster/node-templates/{{ components.clusterapi.workload.node_template.url | basename}}

ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/certauthority.yml

@@ -8,6 +8,34 @@
             combine( components['step-certificates'].chart_values | from_yaml, recursive=True, list_merge='append')
           }}
 
+    - name: Duplicate default provisioner with modified claims
+      ansible.builtin.set_fact:
+        values_new: |
+          {{
+            values_initial |
+              combine({'inject':{'config':{'files':{'ca.json':{'authority': {'provisioners': [
+                values_initial.inject.config.files['ca.json'].authority.provisioners[0] | combine({'name':'long-lived', 'claims':{'maxTLSCertDuration':'87660h'}})
+              ]}}}}}}, list_merge='append_rp', recursive=true)
+          }}
+
+    # We're facing several bugs or niche cases that result in incorrect output, despite being behaviour by design:
+    # - Ansible's `to_yaml` filter, sees `\n` escape sequences in PEM certificate strings and correctly converts them to actual newlines - without any way to prevent this
+    #   So we cannot rely on Ansible to (re)create the helm chart values file
+    # - Python's yaml interpreter sees strings with a value of `y` as short for `yes` or `true`, even when that string is a key name.
+    #   So we cannot use a straightforward yaml document as input for the Ansible helm module (which is written in Python)
+    #
+    # Lets explain the following workaround steps:
+    # - First we convert the dictionary to a json-object (through Ansible), so that yq can read it
+    # - Second we convert the json-object in its entirety to yaml (through yq), so that yq can actually manipulate it.
+    # - Finally, we take one specific subkey's contents (list of dictionaries) and iterate over each with the following steps (with `map`):
+    #   - Convert the dictionary to json with `tojson`
+    #   - Remove newlines (and spaces) with `sub`
+    #   - Remove outer quotes (') with `sed`
+    - name: Save updated values file
+      ansible.builtin.shell:
+        cmd: |
+          echo '{{ values_new | to_nice_json }}' | yq -p json -o yaml | yq e '.inject.config.files["ca.json"].authority.provisioners |= map(tojson | sub("[\n ]";""))' | sed -e "s/- '/- /;s/'$//" > {{ stepconfig.path }}
+
     - name: Install step-ca chart
       kubernetes.core.helm:
         name: step-certificates

ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/main.yml

@@ -1,6 +1,7 @@
 - import_tasks: init.yml
 - import_tasks: k3s.yml
 - import_tasks: assets.yml
+- import_tasks: workflow.yml
 - import_tasks: virtualip.yml
 - import_tasks: metadata.yml
 - import_tasks: storage.yml

ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/workflow.yml

@@ -0,0 +1,54 @@
+- block:
+
+    - name: Create target namespace(s)
+      kubernetes.core.k8s:
+        name: "{{ item }}"
+        kind: Namespace
+        state: present
+        kubeconfig: "{{ kubeconfig.path }}"
+      loop:
+        # - argo-workflows
+        - firstboot
+
+    - name: Create ClusterRoleBinding for default serviceaccount
+      kubernetes.core.k8s:
+        state: present
+        kubeconfig: "{{ kubeconfig.path }}"
+        definition: |
+          kind: ClusterRoleBinding
+          metadata:
+            name: argo-workflows-firstboot-clusteradmin
+          roleRef:
+            apiGroup: rbac.authorization.k8s.io
+            kind: ClusterRole
+            name: cluster-admin
+          subjects:
+          - kind: ServiceAccount
+            name: default
+            namespace: firstboot
+
+    - name: Install argo-workflows chart
+      kubernetes.core.helm:
+        name: argo-workflows
+        chart_ref: /opt/metacluster/helm-charts/argo-workflows
+        release_namespace: argo-workflows
+        create_namespace: true
+        wait: false
+        kubeconfig: "{{ kubeconfig.path }}"
+        values: "{{ components['argo-workflows'].chart_values }}"
+
+    - name: Ensure argo workflows API availability
+      ansible.builtin.uri:
+        url: https://workflow.{{ vapp['metacluster.fqdn'] }}/api/v1/version
+        method: GET
+      register: api_readycheck
+      until:
+        - api_readycheck.json.version is defined
+      retries: "{{ playbook.retries }}"
+      delay: "{{ (storage_benchmark | int) * (playbook.delay.long | int) }}"
+
+  module_defaults:
+    ansible.builtin.uri:
+      validate_certs: no
+      status_code: [200, 201]
+      body_format: json

ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/clusterapi.yml

@@ -127,8 +127,8 @@
     wait: true
     kubeconfig: "{{ kubeconfig.path }}"
   loop:
-    - name: caip-in-cluster-controller-manager
-      namespace: caip-in-cluster-system
+    - name: capi-ipam-in-cluster-controller-manager
+      namespace: capi-ipam-in-cluster-system
     - name: capi-controller-manager
       namespace: capi-system
     - name: capv-controller-manager
@@ -172,7 +172,7 @@
 
 - name: Generate nodepool kustomization manifest
   ansible.builtin.template:
-    src: kustomization.nodepool.j2
+    src: kustomization.longhorn-storage.j2
     dest: "{{ capi_clustermanifest.path }}/kustomization.yaml"
   vars:
     _template:

ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/kustomization.cluster-template.j2

@@ -4,34 +4,6 @@ resources:
 - cluster-template.yaml
 
 patches:
-- patch: |-
-    apiVersion: v1
-    kind: Secret
-    metadata:
-      name: csi-vsphere-config
-      namespace: '${NAMESPACE}'
-    stringData:
-      data: |
-        apiVersion: v1
-        kind: Secret
-        metadata:
-          name: csi-vsphere-config
-          namespace: kube-system
-        stringData:
-          csi-vsphere.conf: |+
-            [Global]
-            insecure-flag = true
-            thumbprint = "${VSPHERE_TLS_THUMBPRINT}"
-            cluster-id = "${NAMESPACE}/${CLUSTER_NAME}"
-
-            [VirtualCenter "${VSPHERE_SERVER}"]
-            user = "${VSPHERE_USERNAME}"
-            password = "${VSPHERE_PASSWORD}"
-            datacenters = "${VSPHERE_DATACENTER}"
-
-            [Network]
-            public-network = "${VSPHERE_NETWORK}"
-        type: Opaque
 - patch: |-
     apiVersion: controlplane.cluster.x-k8s.io/v1beta1
     kind: KubeadmControlPlane
@@ -95,6 +67,7 @@ patches:
     spec:
       template:
         spec:
+          diskGiB: 60
           network:
             devices:
             - dhcp4: false
@@ -114,6 +87,7 @@ patches:
     spec:
       template:
         spec:
+          diskGiB: 60
           network:
             devices:
             - dhcp4: false
@@ -125,6 +99,25 @@ patches:
               - {{ _template.network.dnsserver }}
               networkName: '${VSPHERE_NETWORK}'
 
+- target:
+    group: addons.cluster.x-k8s.io
+    version: v1beta1
+    kind: ClusterResourceSet
+    name: \${CLUSTER_NAME}-crs-0
+  patch: |-
+    - op: replace
+      path: /spec/resources
+      value:
+        - kind: Secret
+          name: cloud-controller-manager
+        - kind: Secret
+          name: cloud-provider-vsphere-credentials
+        - kind: ConfigMap
+          name: cpi-manifests
+    - op: add
+      path: /spec/strategy
+      value: Reconcile
+
 - target:
     group: controlplane.cluster.x-k8s.io
     version: v1beta1
@@ -198,6 +191,8 @@ patches:
     - op: replace
       path: /metadata/name
       value: ${CLUSTER_NAME}-master
+    - op: remove
+      path: /spec/template/spec/thumbprint
 - target:
     group: controlplane.cluster.x-k8s.io
     version: v1beta1
@@ -237,6 +232,8 @@ patches:
     - op: replace
       path: /spec/template/spec/memoryMiB
       value: {{ _template.nodesize.memory }}
+    - op: remove
+      path: /spec/template/spec/thumbprint
 - target:
     group: cluster.x-k8s.io
     version: v1beta1
@@ -258,3 +255,12 @@ patches:
     - op: replace
       path: /metadata/name
       value: ${CLUSTER_NAME}-worker
+
+- target:
+    group: infrastructure.cluster.x-k8s.io
+    version: v1beta1
+    kind: VSphereCluster
+    name: .*
+  patch: |-
+    - op: remove
+      path: /spec/thumbprint

ansible/roles/firstboot/files/ansible_payload/common/roles/workloadcluster/tasks/hypervisor.yml

@@ -11,7 +11,7 @@
     - attribute: cluster
       moref: >-
         $(govc object.collect -json VirtualMachine:{{ moref_id }} | \
-          jq -r '.[] | select(.Name == "runtime").Val.Host | .Type + ":" + .Value')
+          jq -r '.[] | select(.name == "runtime").val.host | .type + ":" + .value')
       part: (NF-1)
     - attribute: datacenter
       moref: VirtualMachine:{{ moref_id }}
@@ -19,27 +19,27 @@
     - attribute: datastore
       moref: >-
         $(govc object.collect -json VirtualMachine:{{ moref_id }} | \
-          jq -r '.[] | select(.Name == "datastore").Val.ManagedObjectReference | .[].Type + ":" + .[].Value')
+          jq -r '.[] | select(.name == "datastore").val._value | .[].type + ":" + .[].value')
       part: NF
     - attribute: folder
       moref: >-
         $(govc object.collect -json VirtualMachine:{{ moref_id }} | \
-          jq -r '.[] | select(.Name == "parent").Val | .Type + ":" + .Value')
+          jq -r '.[] | select(.name == "parent").val | .type + ":" + .value')
       part: 0
     # - attribute: host
     #   moref: >-
     #     $(govc object.collect -json VirtualMachine:{{ moref_id }} | \
-    #       jq -r '.[] | select(.Name == "runtime").Val.Host | .Type + ":" + .Value')
+    #       jq -r '.[] | select(.name == "runtime").val.host | .type + ":" + .value')
     #   part: NF
     - attribute: network
       moref: >-
         $(govc object.collect -json VirtualMachine:{{ moref_id }} | \
-          jq -r '.[] | select(.Name == "network").Val.ManagedObjectReference | .[].Type + ":" + .[].Value')
+          jq -r '.[] | select(.name == "network").val._value | .[].type + ":" + .[].value')
       part: NF
     - attribute: resourcepool
       moref: >-
         $(govc object.collect -json VirtualMachine:{{ moref_id }} | \
-          jq -r '.[] | select(.Name == "resourcePool").Val | .Type + ":" + .Value')
+          jq -r '.[] | select(.name == "resourcePool").val | .type + ":" + .value')
       part: 0
   loop_control:
     label: "{{ item.attribute }}"

ansible/roles/firstboot/files/ansible_payload/upgrade/roles/workloadcluster/tasks/hypervisor.yml

@@ -1,57 +0,0 @@
-- name: Gather hypervisor details
-  ansible.builtin.shell:
-    cmd: govc ls -L {{ item.moref }} | awk -F/ '{print ${{ item.part }}}'
-  environment:
-    GOVC_INSECURE: '1'
-    GOVC_URL: "{{ vapp['hv.fqdn'] }}"
-    GOVC_USERNAME: "{{ vapp['hv.username'] }}"
-    GOVC_PASSWORD: "{{ vapp['hv.password'] }}"
-  register: govc_inventory
-  loop:
-    - attribute: cluster
-      moref: >-
-        $(govc object.collect -json VirtualMachine:{{ moref_id }} | \
-          jq -r '.[] | select(.Name == "runtime").Val.Host | .Type + ":" + .Value')
-      part: (NF-1)
-    - attribute: datacenter
-      moref: VirtualMachine:{{ moref_id }}
-      part: 2
-    - attribute: datastore
-      moref: >-
-        $(govc object.collect -json VirtualMachine:{{ moref_id }} | \
-          jq -r '.[] | select(.Name == "datastore").Val.ManagedObjectReference | .[].Type + ":" + .[].Value')
-      part: NF
-    - attribute: folder
-      moref: >-
-        $(govc object.collect -json VirtualMachine:{{ moref_id }} | \
-          jq -r '.[] | select(.Name == "parent").Val | .Type + ":" + .Value')
-      part: 0
-    # - attribute: host
-    #   moref: >-
-    #     $(govc object.collect -json VirtualMachine:{{ moref_id }} | \
-    #       jq -r '.[] | select(.Name == "runtime").Val.Host | .Type + ":" + .Value')
-    #   part: NF
-    - attribute: network
-      moref: >-
-        $(govc object.collect -json VirtualMachine:{{ moref_id }} | \
-          jq -r '.[] | select(.Name == "network").Val.ManagedObjectReference | .[].Type + ":" + .[].Value')
-      part: NF
-    - attribute: resourcepool
-      moref: >-
-        $(govc object.collect -json VirtualMachine:{{ moref_id }} | \
-          jq -r '.[] | select(.Name == "resourcePool").Val | .Type + ":" + .Value')
-      part: 0
-  loop_control:
-    label: "{{ item.attribute }}"
-
-- name: Retrieve hypervisor TLS thumbprint
-  ansible.builtin.shell:
-    cmd: openssl s_client -connect {{ vapp['hv.fqdn'] }}:443 < /dev/null 2>/dev/null | openssl x509 -fingerprint -noout -in /dev/stdin | awk -F'=' '{print $2}'
-  register: tls_thumbprint
-
-- name: Store hypervisor details in dictionary
-  ansible.builtin.set_fact:
-    vcenter_info: "{{ vcenter_info | default({}) | combine({ item.item.attribute : item.stdout }) }}"
-  loop: "{{ govc_inventory.results }}"
-  loop_control:
-    label: "{{ item.item.attribute }}"

ansible/vars/metacluster.yml

@@ -58,7 +58,8 @@ components:
 
   argo-cd:
     helm:
-      version: 6.7.7  # (=ArgoCD v.2.10.5)
+      # Must match the version referenced at `dependencies.static_binaries[.filename==argo].url`
+      version: 6.7.7  # (=Argo CD v2.10.5)
       chart: argo/argo-cd
       parse_logic: helm template . | yq --no-doc eval '.. | .image? | select(.)' | sort -u | awk '!/ /'
       chart_values: !unsafe |
@@ -79,6 +80,33 @@ components:
           ingress:
             enabled: true
 
+  argo-workflows:
+    helm:
+      version: 0.41.8  # (=Argo Workflows v3.5.7)
+      chart: argo/argo-workflows
+      parse_logic: helm template . | yq --no-doc eval '.. | .image? | select(.)' | sort -u | awk '!/ /'
+      chart_values: !unsafe |
+        # workflow:
+        #   serviceAccount:
+        #     create: true
+        #     name: "argo-workflows"
+        #   rbac:
+        #     create: true
+        controller:
+          workflowNamespaces:
+            - default
+            - firstboot
+        server:
+          authModes:
+            - server
+          ingress:
+            enabled: true
+            hosts:
+              - workflow.{{ vapp['metacluster.fqdn']}}
+            paths:
+              - /
+            pathType: Prefix
+
   cert-manager:
     helm:
       version: 1.14.4
@@ -103,8 +131,8 @@ components:
         calico: v3.27.3
         k8s: v1.30.1
       node_template:
-        # url: https://{{ repo_username }}:{{ repo_password }}@sn.itch.fyi/Repository/rel/ubuntu-2204-kube-v1.27.1.ova
-        url: https://github.com/kubernetes-sigs/cluster-api-provider-vsphere/releases/download/templates%2Fv1.30.0/ubuntu-2204-kube-v1.30.0.ova
+      # Not used anymore; should be uploaded to hypervisor manually!
+      # https://github.com/kubernetes-sigs/cluster-api-provider-vsphere/releases/download/templates%2Fv1.30.0/
 
   dex:
     helm:
@@ -336,6 +364,8 @@ dependencies:
     - registry.k8s.io/sig-storage/livenessprobe:v2.10.0
 
   static_binaries:
+    - filename: argo
+      url: https://github.com/argoproj/argo-workflows/releases/download/v3.5.7/argo-linux-amd64.gz
     - filename: clusterctl
       url: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.6.3/clusterctl-linux-amd64
     - filename: govc

deployment/playbook.yml

@@ -0,0 +1,78 @@
+- hosts: localhost
+  vars_files:
+    - vars/ova.bootstrap.yaml
+    - vars/hv.vcenter.yaml
+    - vars/pb.secrets.yaml
+  tasks:
+
+  - name: Retrieve target folder details
+    community.vmware.vmware_vm_info:
+      hostname: "{{ hv.hostname }}"
+      username: "{{ hv.username }}"
+      password: "{{ secrets.hv.password }}"
+      folder: "{{ hv.folder }}"
+      validate_certs: false
+    register: vm_info
+
+  - name: User prompt
+    ansible.builtin.pause:
+      prompt: Virtual machine '{{ appliance.id }}' already exists. Delete to continue [yes] or abort [no]?"
+    register: prompt
+    until:
+      - prompt.user_input in ['yes', 'no']
+    delay: 0
+    when: (vm_info | selectattr('guest_name', 'equalto', appliance.id) | length) > 0
+
+  - name: Destroy existing VM
+    community.vmware.vmware_guest:
+      hostname: "{{ hv.hostname }}"
+      username: "{{ hv.username }}"
+      password: "{{ secrets.hv.password }}"
+      folder: "{{ hv.folder }}"
+      name: appliance.id
+      state: absent
+    when:
+      - (vm_info | selectattr('guest_name', 'equalto', appliance.id) | length) > 0
+      - (prompt.user_input | bool) == true
+
+  - name: Deploy VM from OVA-template
+    community.vmware.vmware_deploy_ovf:
+      hostname: "{{ hv.hostname }}"
+      username: "{{ hv.username }}"
+      password: "{{ secrets.hv.password }}"
+      validate_certs: false
+      datacenter: "{{ hv.datacenter }}"
+      folder: "{{ hv.folder }}"
+      cluster: "{{ hv.cluster }}"
+      name: airgapped-k8s-meta1
+      datastore: "{{ hv.datastore }}"
+      disk_provisioning: thin
+      networks:
+        "LAN": "{{ hv.network }}"
+      power_on: yes
+      ovf: "{{ appliance.path }}/{{ appliance.filename }}"
+      deployment_option: cp1w1ws0
+      properties:
+        metacluster.fqdn: k8s.lab
+        metacluster.vip: 192.168.154.125
+        metacluster.token: "{{ secrets.appliance.installtoken }}"
+        # guestinfo.hostname: _default
+        metacluster.password: "{{ secrets.appliance.password }}"
+        guestinfo.ipaddress: 192.168.154.126
+        guestinfo.prefixlength: '24'
+        guestinfo.dnsserver: 192.168.154.225
+        guestinfo.gateway: 192.168.154.1
+        # workloadcluster.name: _default
+        workloadcluster.vip: 192.168.154.130
+        ippool.startip: 192.168.154.135
+        ippool.endip: 192.168.154.140
+        workloadcluster.nodetemplate: ubuntu-2204-kube-v1.30.0
+        workloadcluster.nodesize: small
+        # workloadcluster.additionaldisk: '75'
+        guestinfo.rootsshkey: ssh-rsa AAAAB3NzaC1yc2EAAAABJQAAAQEAiRc7Og+cRJGFwdUzgpX9YqvVenTk54N4kqM7emEfYHdsJLMjKQyxr8hklHmsam5dzxx3itFzc6SLf/ldJJ2JZuzE5FiCqUXXv4UFwN6HF5xqn7PTLicvWZH93H4m1gOlD5Dfzi4Es34v5zRBwbMScOgekk/LweTgl35jGKDgMP5DjGTqkPf7Ndh9+iuQrz99JEr8egl3bj+jIlKjScfaQbbnu3AJIRwZwTKgw0AOkLliQdEPNLvG5/ZImxJG4oHV9/uNkfdJObLjT1plR1HbVNskV5fuRNE/vnUiWl9jAJ1RT83GOqV0sQ+Q7p214fkgqb3JPvci/s0Bb7RA85hBEQ== bessems.eu
+        hv.fqdn: "{{ hv.hostname }}"
+        hv.username: "{{ hv.username }}"
+        hv.password: "{{ secrets.hv.password }}"
+        ldap.fqdn: _unused
+        ldap.dn: _unused
+        ldap.password: _unused

deployment/requirements.yaml

@@ -0,0 +1,5 @@
+collections:
+# - ansible.posix
+# - ansible.utils
+# - community.general
+- community.vmware