From d67bf86dab5c3452da2b2ed043567616f6c551a6 Mon Sep 17 00:00:00 2001 From: Danny Bessems Date: Sat, 21 Jan 2023 16:12:11 +0100 Subject: [PATCH] Test ansible output regression workaround #2;Refactor vapp properties;Add kube-vip dependency;Refactor netplan;Download/Trust root CA --- ansible/roles/assets/tasks/main.yml | 1 + ansible/roles/assets/tasks/manifests.yml | 18 +++++- .../bootstrap/roles/metacluster/tasks/k3s.yml | 2 +- .../roles/metacluster/tasks/kube-vip.yml | 27 +++++++++ .../roles/metacluster/tasks/main.yml | 1 + .../common/roles/network/tasks/main.yml | 7 +++ .../common/roles/network/templates/netplan.j2 | 11 ++-- .../upgrade/roles/metacluster/tasks/init.yml | 30 ++++++++++ .../upgrade/roles/metacluster/tasks/main.yml | 2 +- .../roles/firstboot/templates/firstboot.j2 | 2 +- ansible/vars/metacluster.yml | 5 ++ scripts/Update-OvfConfiguration.bootstrap.yml | 35 ++++++++---- scripts/Update-OvfConfiguration.upgrade.yml | 56 +++++++------------ 13 files changed, 141 insertions(+), 56 deletions(-) create mode 100644 ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/kube-vip.yml create mode 100644 ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/init.yml diff --git a/ansible/roles/assets/tasks/main.yml b/ansible/roles/assets/tasks/main.yml index a1258d5..ed3d916 100644 --- a/ansible/roles/assets/tasks/main.yml +++ b/ansible/roles/assets/tasks/main.yml @@ -15,6 +15,7 @@ - /opt/metacluster/git-repositories/gitops - /opt/metacluster/helm-charts - /opt/metacluster/k3s + - /opt/metacluster/kube-vip - /opt/workloadcluster/node-templates - /var/lib/rancher/k3s/agent/images - /var/lib/rancher/k3s/server/manifests diff --git a/ansible/roles/assets/tasks/manifests.yml b/ansible/roles/assets/tasks/manifests.yml index af0e1da..c57be4a 100644 --- a/ansible/roles/assets/tasks/manifests.yml +++ b/ansible/roles/assets/tasks/manifests.yml @@ -11,7 +11,14 @@ - name: Write dict to vars_file ansible.builtin.copy: dest: /opt/firstboot/ansible/vars/metacluster.yml - content: "{{ { 'components': (chart_values | combine({ 'clusterapi': components.clusterapi })) } | to_nice_yaml(indent=2, width=4096) }}" + content: >- + {{ + { 'components': ( + chart_values | + combine({ 'clusterapi': components.clusterapi }) | + combine({ 'kubevip' : components.kubevip }) ) + } | to_nice_yaml(indent=2, width=4096) + }} - name: Download ClusterAPI manifests ansible.builtin.get_url: @@ -58,6 +65,15 @@ delay: 5 until: clusterapi_manifests is not failed +- name: Download kube-vip RBAC manifest + ansible.builtin.get_url: + url: https://kube-vip.io/manifests/rbac.yaml + dest: /opt/metacluster/kube-vip/rbac.yaml + register: clusterapi_manifest + retries: 5 + delay: 5 + until: kubevip_manifest is not failed + # - name: Inject manifests # ansible.builtin.template: # src: "{{ item.type }}.j2" diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/k3s.yml b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/k3s.yml index 56e97fc..05a55a3 100644 --- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/k3s.yml +++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/k3s.yml @@ -27,7 +27,7 @@ chdir: /opt/metacluster/k3s environment: INSTALL_K3S_SKIP_DOWNLOAD: 'true' - INSTALL_K3S_EXEC: "server --cluster-init --token {{ vapp['metacluster.token'] }} --disable local-storage --config /etc/rancher/k3s/config.yaml" + INSTALL_K3S_EXEC: "server --cluster-init --token {{ vapp['metacluster.token'] }} --tls-san {{ vapp['metacluster.vip'] }} --disable local-storage --config /etc/rancher/k3s/config.yaml" when: ansible_facts.services['k3s.service'] is undefined - name: Debug possible taints on k3s node diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/kube-vip.yml b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/kube-vip.yml new file mode 100644 index 0000000..86ca25a --- /dev/null +++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/kube-vip.yml @@ -0,0 +1,27 @@ +- name: Generate kube-vip manifest + ansible.builtin.command: + cmd: >- + ctr run --rm --net-host ghcr.io/kube-vip/kube-vip:{{ components.kubevip.version }} vip \ + /kube-vip manifest daemonset \ + --interface eth0 \ + --address {{ vapp['metacluster.vip'] }} \ + --inCluster \ + --taint \ + --controlplane \ + --services \ + --arp \ + --leaderElection + register: kubevip_manifest + +- name: Inject manifests + ansible.builtin.copy: + dest: /var/lib/rancher/k3s/server/manifests/kubevip-manifest.yaml + content: >- + {{ lookup('ansible.builtin.file', '/opt/metacluster/kube-vip/rbac.yaml') }} + --- + {{ kubevip_manifest.stdout }} + notify: + - Apply manifests + +- name: Trigger handlers + ansible.builtin.meta: flush_handlers diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/main.yml b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/main.yml index 5928850..8887e0e 100644 --- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/main.yml +++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/main.yml @@ -1,5 +1,6 @@ - import_tasks: init.yml - import_tasks: k3s.yml +- import_tasks: kube-vip.yml - import_tasks: assets.yml - import_tasks: ingress.yml - import_tasks: storage.yml diff --git a/ansible/roles/firstboot/files/ansible_payload/common/roles/network/tasks/main.yml b/ansible/roles/firstboot/files/ansible_payload/common/roles/network/tasks/main.yml index 1393ce0..cde402b 100644 --- a/ansible/roles/firstboot/files/ansible_payload/common/roles/network/tasks/main.yml +++ b/ansible/roles/firstboot/files/ansible_payload/common/roles/network/tasks/main.yml @@ -6,6 +6,13 @@ ansible.builtin.template: src: netplan.j2 dest: /etc/netplan/00-installer-config.yaml + vars: + _template: + macaddress: "{{ ansible_facts.ansible_facts.default_ipv4.macaddress }}" + ipaddress: "{{ vapp['guestinfo.ipaddress'] }}" + prefixlength: "{{ vapp['guestinfo.prefixlength'] }}" + gateway: "{{ vapp['guestinfo.gateway'] }}" + dnsserver: "{{ vapp['guestinfo.dnsserver'] }}" - name: Apply netplan configuration ansible.builtin.shell: diff --git a/ansible/roles/firstboot/files/ansible_payload/common/roles/network/templates/netplan.j2 b/ansible/roles/firstboot/files/ansible_payload/common/roles/network/templates/netplan.j2 index 7500090..3a32561 100644 --- a/ansible/roles/firstboot/files/ansible_payload/common/roles/network/templates/netplan.j2 +++ b/ansible/roles/firstboot/files/ansible_payload/common/roles/network/templates/netplan.j2 @@ -1,10 +1,13 @@ network: version: 2 ethernets: - ens192: + id0: + set-name: eth0 + match: + macaddress: {{ _template.macaddress }} addresses: - - {{ vapp['guestinfo.ipaddress'] }}/{{ vapp['guestinfo.prefixlength'] }} - gateway4: {{ vapp['guestinfo.gateway'] }} + - {{ _template.ipaddress }}/{{ _template.prefixlength }} + gateway4: {{ _template.gateway }} nameservers: addresses: - - {{ vapp['guestinfo.dnsserver'] }} + - {{ _template.dnsserver }} diff --git a/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/init.yml b/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/init.yml new file mode 100644 index 0000000..1106194 --- /dev/null +++ b/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/init.yml @@ -0,0 +1,30 @@ +- name: Configure fallback name resolution + ansible.builtin.lineinfile: + path: /etc/hosts + line: "{{ vapp['metacluster.vip'] }} {{ item + '.' + vapp['metacluster.fqdn'] }}" + state: present + loop: + # TODO: Make this list dynamic + - ca + - git + - gitops + - ingress + - registry + - storage + +- name: Retrieve root CA certificate + ansible.builtin.uri: + url: https://ca.{{ vapp['metacluster.fqdn'] }}/roots + validate_certs: no + method: GET + status_code: [200, 201] + register: rootca_certificate + +- name: Store root CA certificate + ansible.builtin.copy: + dest: /usr/local/share/ca-certificates/root_ca.crt + content: "{{ rootca_certificate.json.crts | list | join('\n') }}" + +- name: Update certificate truststore + ansible.builtin.command: + cmd: update-ca-certificates diff --git a/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/main.yml b/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/main.yml index d35e121..bad0a94 100644 --- a/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/main.yml +++ b/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/main.yml @@ -1,4 +1,4 @@ -# - import_tasks: init.yml +- import_tasks: init.yml - import_tasks: k3s.yml # - import_tasks: assets.yml # - import_tasks: ingress.yml diff --git a/ansible/roles/firstboot/templates/firstboot.j2 b/ansible/roles/firstboot/templates/firstboot.j2 index 19b5588..b028cd4 100644 --- a/ansible/roles/firstboot/templates/firstboot.j2 +++ b/ansible/roles/firstboot/templates/firstboot.j2 @@ -1,4 +1,4 @@ #!/bin/bash # Apply firstboot configuration w/ ansible -/usr/local/bin/ansible-playbook /opt/firstboot/ansible/playbook.yml | tee -a /var/log/firstboot.log > /dev/tty1 2>&1 +/usr/local/bin/ansible-playbook -e "PYTHONUNBUFFERED=1" /opt/firstboot/ansible/playbook.yml | tee -a /var/log/firstboot.log > /dev/tty1 2>&1 diff --git a/ansible/vars/metacluster.yml b/ansible/vars/metacluster.yml index 521c282..eabf0c7 100644 --- a/ansible/vars/metacluster.yml +++ b/ansible/vars/metacluster.yml @@ -148,6 +148,10 @@ components: registry: size: 25Gi + kubevip: + Must match the version referenced at `dependencies.container_images` + version: v0.5.8 + longhorn: helm: version: 1.4.0 @@ -203,6 +207,7 @@ dependencies: - kubernetes.core container_images: + - ghcr.io/kube-vip/kube-vip:v0.5.8 # The following list is generated by running the following commands: # $ clusterctl init -i vsphere: [...] # $ clusterctl generate cluster [...] | yq eval '.data.data' | yq --no-doc eval '.. | .image? | select(.)' | sort -u diff --git a/scripts/Update-OvfConfiguration.bootstrap.yml b/scripts/Update-OvfConfiguration.bootstrap.yml index 339748d..306dcf3 100644 --- a/scripts/Update-OvfConfiguration.bootstrap.yml +++ b/scripts/Update-OvfConfiguration.bootstrap.yml @@ -15,7 +15,7 @@ DynamicDisks: Constraints: Minimum: 100 Maximum: '' - PropertyCategory: 1 + PropertyCategory: 2 PropertyCategories: @@ -31,6 +31,22 @@ PropertyCategories: - Name: 1) Meta-cluster ProductProperties: + - Key: metacluster.fqdn + Type: string(1..) + Label: Meta-cluster FQDN* + Description: Respective subdomains will be available for each component (e.g. storage.example.org); this address should already be configured as a wildcard record within your DNS zone. + DefaultValue: meta.k8s.cluster + Configurations: '*' + UserConfigurable: true + + - key: metacluster.vip + Type: ip + Label: Meta-cluster virtual IP* + Description: Meta-cluster control plane endpoint virtual IP + DefaultValue: '' + Configurations: '*' + UserConfigurable: true + - key: metacluster.token Type: string(1..) Label: K3s install token* @@ -39,6 +55,9 @@ PropertyCategories: Configurations: '*' UserConfigurable: true +- Name: 2) Meta-cluster initial node + ProductProperties: + - Key: guestinfo.hostname Type: string(1..15) Label: Hostname* @@ -71,14 +90,6 @@ PropertyCategories: Configurations: '*' UserConfigurable: true - - Key: metacluster.fqdn - Type: string(1..) - Label: Appliance FQDN* - Description: Respective subdomains will be available for each component (e.g. storage.example.org); this address should already be configured as a wildcard record within your DNS zone. - DefaultValue: meta.k8s.cluster - Configurations: '*' - UserConfigurable: true - - Key: guestinfo.dnsserver Type: ip Label: DNS server* @@ -103,7 +114,7 @@ PropertyCategories: Configurations: '*' UserConfigurable: true -- Name: 2) Workload-cluster +- Name: 3) Workload-cluster ProductProperties: - Key: workloadcluster.name @@ -138,7 +149,7 @@ PropertyCategories: Configurations: '*' UserConfigurable: true -- Name: 3) Common +- Name: 4) Common ProductProperties: - Key: guestinfo.rootsshkey @@ -149,7 +160,7 @@ PropertyCategories: Configurations: '*' UserConfigurable: true -- Name: 4) Hypervisor +- Name: 5) Hypervisor ProductProperties: - Key: hv.fqdn diff --git a/scripts/Update-OvfConfiguration.upgrade.yml b/scripts/Update-OvfConfiguration.upgrade.yml index a7eedf6..89dd49f 100644 --- a/scripts/Update-OvfConfiguration.upgrade.yml +++ b/scripts/Update-OvfConfiguration.upgrade.yml @@ -1,13 +1,3 @@ -DeploymentConfigurations: - -- Id: cp1w1 - Label: 'Workload-cluster: 1 control-plane node/1 worker node' - Description: 1 control-plane node/1 worker node - -- Id: cp1w2 - Label: 'Workload-cluster: 1 control-plane node/2 worker nodes' - Description: 1 control-plane node/2 worker nodes - DynamicDisks: - Description: Longhorn persistent storage @@ -19,16 +9,26 @@ DynamicDisks: PropertyCategories: -- Name: 0) Deployment information +- Name: 1) Existing meta-cluster ProductProperties: - - Key: deployment.type - Type: string - Value: - - cp1w1 - - cp1w2 - UserConfigurable: false -- Name: 1) Meta-cluster + - Key: metacluster.vip + Type: string(1..) + Label: Meta-cluster FQDN/IP-address* + Description: The address of the target meta-cluster which this appliance will perform an upgrade on. + DefaultValue: '' + Configurations: '*' + UserConfigurable: true + + - key: metacluster.token + Type: string(1..) + Label: K3s install token* + Description: Must match the token originally used for the target meta-cluster + DefaultValue: '' + Configurations: '*' + UserConfigurable: true + +- Name: 2) Add meta-cluster node ProductProperties: - Key: guestinfo.hostname @@ -63,22 +63,6 @@ PropertyCategories: Configurations: '*' UserConfigurable: true - - Key: metacluster.fqdn - Type: string(1..) - Label: Meta-cluster FQDN/IP-address* - Description: The address of the target meta-cluster which this appliance will perform an upgrade on. - DefaultValue: '' - Configurations: '*' - UserConfigurable: true - - - key: metacluster.token - Type: string(1..) - Label: K3s install token* - Description: Must match the token originally used for the target meta-cluster - DefaultValue: '' - Configurations: '*' - UserConfigurable: true - - Key: guestinfo.dnsserver Type: ip Label: DNS server* @@ -103,7 +87,7 @@ PropertyCategories: Configurations: '*' UserConfigurable: true -- Name: 2) Common +- Name: 3) Common ProductProperties: - Key: guestinfo.rootsshkey @@ -114,7 +98,7 @@ PropertyCategories: Configurations: '*' UserConfigurable: true -- Name: 3) Hypervisor +- Name: 4) Hypervisor ProductProperties: - Key: hv.fqdn