Test ansible output regression workaround #2;Refactor vapp properties;Add kube-vip dependency;Refactor netplan;Download/Trust root CA

ansible/roles/assets/tasks/main.yml

@@ -15,6 +15,7 @@
     - /opt/metacluster/git-repositories/gitops
     - /opt/metacluster/helm-charts
     - /opt/metacluster/k3s
+    - /opt/metacluster/kube-vip
     - /opt/workloadcluster/node-templates
     - /var/lib/rancher/k3s/agent/images
     - /var/lib/rancher/k3s/server/manifests

ansible/roles/assets/tasks/manifests.yml

@@ -11,7 +11,14 @@
     - name: Write dict to vars_file
       ansible.builtin.copy:
         dest: /opt/firstboot/ansible/vars/metacluster.yml
-        content: "{{ { 'components': (chart_values | combine({ 'clusterapi': components.clusterapi })) } | to_nice_yaml(indent=2, width=4096) }}"
+        content: >-
+          {{
+            { 'components': (
+              chart_values |
+              combine({ 'clusterapi': components.clusterapi }) |
+              combine({ 'kubevip'   : components.kubevip }) )
+            } | to_nice_yaml(indent=2, width=4096)
+          }}
 
 - name: Download ClusterAPI manifests
   ansible.builtin.get_url:
@@ -58,6 +65,15 @@
   delay: 5
   until: clusterapi_manifests is not failed
 
+- name: Download kube-vip RBAC manifest
+  ansible.builtin.get_url:
+    url: https://kube-vip.io/manifests/rbac.yaml
+    dest: /opt/metacluster/kube-vip/rbac.yaml
+  register: clusterapi_manifest
+  retries: 5
+  delay: 5
+  until: kubevip_manifest is not failed
+
 # - name: Inject manifests
 #   ansible.builtin.template:
 #     src: "{{ item.type }}.j2"

ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/k3s.yml

@@ -27,7 +27,7 @@
     chdir: /opt/metacluster/k3s
   environment:
     INSTALL_K3S_SKIP_DOWNLOAD: 'true'
-    INSTALL_K3S_EXEC: "server --cluster-init --token {{ vapp['metacluster.token'] }} --disable local-storage --config /etc/rancher/k3s/config.yaml"
+    INSTALL_K3S_EXEC: "server --cluster-init --token {{ vapp['metacluster.token'] }} --tls-san {{ vapp['metacluster.vip'] }} --disable local-storage --config /etc/rancher/k3s/config.yaml"
   when: ansible_facts.services['k3s.service'] is undefined
 
 - name: Debug possible taints on k3s node

ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/kube-vip.yml

@@ -0,0 +1,27 @@
+- name: Generate kube-vip manifest
+  ansible.builtin.command:
+    cmd: >-
+      ctr run --rm --net-host ghcr.io/kube-vip/kube-vip:{{ components.kubevip.version }} vip \
+        /kube-vip manifest daemonset \
+          --interface eth0 \
+          --address {{ vapp['metacluster.vip'] }} \
+          --inCluster \
+          --taint \
+          --controlplane \
+          --services \
+          --arp \
+          --leaderElection
+  register: kubevip_manifest
+
+- name: Inject manifests
+  ansible.builtin.copy:
+    dest: /var/lib/rancher/k3s/server/manifests/kubevip-manifest.yaml
+    content: >-
+      {{ lookup('ansible.builtin.file', '/opt/metacluster/kube-vip/rbac.yaml') }}
+      ---
+      {{ kubevip_manifest.stdout }}
+  notify:
+    - Apply manifests
+
+- name: Trigger handlers
+  ansible.builtin.meta: flush_handlers

ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/main.yml

@@ -1,5 +1,6 @@
 - import_tasks: init.yml
 - import_tasks: k3s.yml
+- import_tasks: kube-vip.yml
 - import_tasks: assets.yml
 - import_tasks: ingress.yml
 - import_tasks: storage.yml

ansible/roles/firstboot/files/ansible_payload/common/roles/network/tasks/main.yml

@@ -6,6 +6,13 @@
   ansible.builtin.template:
     src: netplan.j2
     dest: /etc/netplan/00-installer-config.yaml
+  vars:
+    _template:
+      macaddress: "{{ ansible_facts.ansible_facts.default_ipv4.macaddress }}"
+      ipaddress: "{{ vapp['guestinfo.ipaddress'] }}"
+      prefixlength: "{{ vapp['guestinfo.prefixlength'] }}"
+      gateway: "{{ vapp['guestinfo.gateway'] }}"
+      dnsserver: "{{ vapp['guestinfo.dnsserver'] }}"
 
 - name: Apply netplan configuration
   ansible.builtin.shell:

ansible/roles/firstboot/files/ansible_payload/common/roles/network/templates/netplan.j2

@@ -1,10 +1,13 @@
 network:
   version: 2
   ethernets:
-    ens192:
+    id0:
+      set-name: eth0
+      match:
+        macaddress: {{ _template.macaddress }}
       addresses:
-      - {{ vapp['guestinfo.ipaddress'] }}/{{ vapp['guestinfo.prefixlength'] }}
+      - {{ _template.ipaddress }}/{{ _template.prefixlength }}
-      gateway4: {{ vapp['guestinfo.gateway'] }}
+      gateway4: {{ _template.gateway }}
       nameservers:
         addresses:
-        - {{ vapp['guestinfo.dnsserver'] }}
+        - {{ _template.dnsserver }}

ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/init.yml

@@ -0,0 +1,30 @@
+- name: Configure fallback name resolution
+  ansible.builtin.lineinfile:
+    path: /etc/hosts
+    line: "{{ vapp['metacluster.vip'] }}  {{ item + '.' + vapp['metacluster.fqdn'] }}"
+    state: present
+  loop:
+    # TODO: Make this list dynamic
+    - ca
+    - git
+    - gitops
+    - ingress
+    - registry
+    - storage
+
+- name: Retrieve root CA certificate
+  ansible.builtin.uri:
+    url: https://ca.{{ vapp['metacluster.fqdn'] }}/roots
+    validate_certs: no
+    method: GET
+    status_code: [200, 201]
+  register: rootca_certificate
+
+- name: Store root CA certificate
+  ansible.builtin.copy:
+    dest: /usr/local/share/ca-certificates/root_ca.crt
+    content: "{{ rootca_certificate.json.crts | list | join('\n') }}"
+
+- name: Update certificate truststore
+  ansible.builtin.command:
+    cmd: update-ca-certificates

ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/main.yml

@@ -1,4 +1,4 @@
-# - import_tasks: init.yml
+- import_tasks: init.yml
 - import_tasks: k3s.yml
 # - import_tasks: assets.yml
 # - import_tasks: ingress.yml

ansible/roles/firstboot/templates/firstboot.j2

@@ -1,4 +1,4 @@
 #!/bin/bash
 
 # Apply firstboot configuration w/ ansible
-/usr/local/bin/ansible-playbook /opt/firstboot/ansible/playbook.yml | tee -a /var/log/firstboot.log > /dev/tty1 2>&1
+/usr/local/bin/ansible-playbook -e "PYTHONUNBUFFERED=1" /opt/firstboot/ansible/playbook.yml | tee -a /var/log/firstboot.log > /dev/tty1 2>&1

ansible/vars/metacluster.yml

@@ -148,6 +148,10 @@ components:
             registry:
               size: 25Gi
 
+  kubevip:
+    Must match the version referenced at `dependencies.container_images`
+    version: v0.5.8
+
   longhorn:
     helm:
       version: 1.4.0
@@ -203,6 +207,7 @@ dependencies:
     - kubernetes.core
 
   container_images:
+    - ghcr.io/kube-vip/kube-vip:v0.5.8
     # The following list is generated by running the following commands:
     #   $ clusterctl init -i vsphere:<version> [...]
     #   $ clusterctl generate cluster <name> [...] | yq eval '.data.data' | yq --no-doc eval '.. | .image? | select(.)' | sort -u

scripts/Update-OvfConfiguration.bootstrap.yml

@@ -15,7 +15,7 @@ DynamicDisks:
   Constraints:
     Minimum: 100
     Maximum: ''
-  PropertyCategory: 1
+  PropertyCategory: 2
 
 PropertyCategories:
 
@@ -31,6 +31,22 @@ PropertyCategories:
 - Name: 1) Meta-cluster
   ProductProperties:
 
+  - Key: metacluster.fqdn
+    Type: string(1..)
+    Label: Meta-cluster FQDN*
+    Description: Respective subdomains will be available for each component (e.g. storage.example.org); this address should already be configured as a wildcard record within your DNS zone.
+    DefaultValue: meta.k8s.cluster
+    Configurations: '*'
+    UserConfigurable: true
+
+  - key: metacluster.vip
+    Type: ip
+    Label: Meta-cluster virtual IP*
+    Description: Meta-cluster control plane endpoint virtual IP
+    DefaultValue: ''
+    Configurations: '*'
+    UserConfigurable: true
+
   - key: metacluster.token
     Type: string(1..)
     Label: K3s install token*
@@ -39,6 +55,9 @@ PropertyCategories:
     Configurations: '*'
     UserConfigurable: true
 
+- Name: 2) Meta-cluster initial node
+  ProductProperties:
+
   - Key: guestinfo.hostname
     Type: string(1..15)
     Label: Hostname*
@@ -71,14 +90,6 @@ PropertyCategories:
     Configurations: '*'
     UserConfigurable: true
 
-  - Key: metacluster.fqdn
-    Type: string(1..)
-    Label: Appliance FQDN*
-    Description: Respective subdomains will be available for each component (e.g. storage.example.org); this address should already be configured as a wildcard record within your DNS zone.
-    DefaultValue: meta.k8s.cluster
-    Configurations: '*'
-    UserConfigurable: true
-
   - Key: guestinfo.dnsserver
     Type: ip
     Label: DNS server*
@@ -103,7 +114,7 @@ PropertyCategories:
     Configurations: '*'
     UserConfigurable: true
 
-- Name: 2) Workload-cluster
+- Name: 3) Workload-cluster
   ProductProperties:
 
   - Key: workloadcluster.name
@@ -138,7 +149,7 @@ PropertyCategories:
     Configurations: '*'
     UserConfigurable: true
 
-- Name: 3) Common
+- Name: 4) Common
   ProductProperties:
 
   - Key: guestinfo.rootsshkey
@@ -149,7 +160,7 @@ PropertyCategories:
     Configurations: '*'
     UserConfigurable: true
 
-- Name: 4) Hypervisor
+- Name: 5) Hypervisor
   ProductProperties:
 
   - Key: hv.fqdn

scripts/Update-OvfConfiguration.upgrade.yml

@@ -1,13 +1,3 @@
-DeploymentConfigurations:
-
-- Id: cp1w1
-  Label: 'Workload-cluster: 1 control-plane node/1 worker node'
-  Description: 1 control-plane node/1 worker node
-
-- Id: cp1w2
-  Label: 'Workload-cluster: 1 control-plane node/2 worker nodes'
-  Description: 1 control-plane node/2 worker nodes
-
 DynamicDisks:
 
 - Description: Longhorn persistent storage
@@ -19,16 +9,26 @@ DynamicDisks:
 
 PropertyCategories:
 
-- Name: 0) Deployment information
+- Name: 1) Existing meta-cluster
   ProductProperties:
-  - Key: deployment.type
-    Type: string
-    Value:
-    - cp1w1
-    - cp1w2
-    UserConfigurable: false
 
-- Name: 1) Meta-cluster
+  - Key: metacluster.vip
+    Type: string(1..)
+    Label: Meta-cluster FQDN/IP-address*
+    Description: The address of the target meta-cluster which this appliance will perform an upgrade on.
+    DefaultValue: ''
+    Configurations: '*'
+    UserConfigurable: true
+
+  - key: metacluster.token
+    Type: string(1..)
+    Label: K3s install token*
+    Description: Must match the token originally used for the target meta-cluster
+    DefaultValue: ''
+    Configurations: '*'
+    UserConfigurable: true
+
+- Name: 2) Add meta-cluster node
   ProductProperties:
 
   - Key: guestinfo.hostname
@@ -63,22 +63,6 @@ PropertyCategories:
     Configurations: '*'
     UserConfigurable: true
 
-  - Key: metacluster.fqdn
-    Type: string(1..)
-    Label: Meta-cluster FQDN/IP-address*
-    Description: The address of the target meta-cluster which this appliance will perform an upgrade on.
-    DefaultValue: ''
-    Configurations: '*'
-    UserConfigurable: true
-
-  - key: metacluster.token
-    Type: string(1..)
-    Label: K3s install token*
-    Description: Must match the token originally used for the target meta-cluster
-    DefaultValue: ''
-    Configurations: '*'
-    UserConfigurable: true
-
   - Key: guestinfo.dnsserver
     Type: ip
     Label: DNS server*
@@ -103,7 +87,7 @@ PropertyCategories:
     Configurations: '*'
     UserConfigurable: true
 
-- Name: 2) Common
+- Name: 3) Common
   ProductProperties:
 
   - Key: guestinfo.rootsshkey
@@ -114,7 +98,7 @@ PropertyCategories:
     Configurations: '*'
     UserConfigurable: true
 
-- Name: 3) Hypervisor
+- Name: 4) Hypervisor
   ProductProperties:
 
   - Key: hv.fqdn