From f74d94a5e0ac9af98182e8cdd5aa2e6cd2ca17ec Mon Sep 17 00:00:00 2001 From: Danny Bessems Date: Fri, 3 Feb 2023 13:11:54 +0100 Subject: [PATCH] Update hypervisor details;Upgrade components;Housekeeping;Add decom tasks;Prevent configuration reset #2;Add morefid label --- .../roles/metacluster/tasks/certauthority.yml | 4 +-- .../bootstrap/roles/metacluster/tasks/git.yml | 4 +-- .../roles/metacluster/tasks/gitops.yml | 4 +-- .../bootstrap/roles/metacluster/tasks/k3s.yml | 10 +++++++ .../roles/metacluster/tasks/registry.yml | 4 +-- .../roles/metacluster/tasks/storage.yml | 4 +-- .../workloadcluster/tasks/clusterapi.yml | 4 +-- .../ansible_payload/upgrade/playbook.yml | 1 + .../upgrade/roles/decommission/tasks/k3s.yml | 24 +++++++++++++++++ .../upgrade/roles/decommission/tasks/main.yml | 18 +++++++++++++ .../roles/decommission/tasks/storage.yml | 26 ++++++++++++++++++ .../decommission/tasks/virtualmachine.yml | 27 +++++++++++++++++++ .../upgrade/roles/metacluster/tasks/init.yml | 5 ++++ .../upgrade/roles/metacluster/tasks/k3s.yml | 10 +++++++ .../roles/metacluster/tasks/registry.yml | 4 +-- .../roles/metacluster/tasks/storage.yml | 6 ++--- ansible/vars/metacluster.yml | 4 +-- packer/vsphere.auto.pkrvars.hcl | 4 +-- 18 files changed, 142 insertions(+), 21 deletions(-) create mode 100644 ansible/roles/firstboot/files/ansible_payload/upgrade/roles/decommission/tasks/k3s.yml create mode 100644 ansible/roles/firstboot/files/ansible_payload/upgrade/roles/decommission/tasks/main.yml create mode 100644 ansible/roles/firstboot/files/ansible_payload/upgrade/roles/decommission/tasks/storage.yml create mode 100644 ansible/roles/firstboot/files/ansible_payload/upgrade/roles/decommission/tasks/virtualmachine.yml diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/certauthority.yml b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/certauthority.yml index 9fc7707..e2da602 100644 --- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/certauthority.yml +++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/certauthority.yml @@ -16,9 +16,9 @@ name: step-certificates chart_ref: /opt/metacluster/helm-charts/step-certificates release_namespace: step-ca - create_namespace: yes + create_namespace: true # Unable to use REST api based readycheck due to lack of ingress - wait: yes + wait: true kubeconfig: "{{ kubeconfig.path }}" values_files: - "{{ values_file.path }}" diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/git.yml b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/git.yml index c1f875a..e10f19a 100644 --- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/git.yml +++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/git.yml @@ -5,8 +5,8 @@ name: gitea chart_ref: /opt/metacluster/helm-charts/gitea release_namespace: gitea - create_namespace: yes - wait: no + create_namespace: true + wait: false kubeconfig: "{{ kubeconfig.path }}" values: "{{ components.gitea.chart_values }}" diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/gitops.yml b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/gitops.yml index 2e1eb23..8155cb7 100644 --- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/gitops.yml +++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/gitops.yml @@ -5,8 +5,8 @@ name: argo-cd chart_ref: /opt/metacluster/helm-charts/argo-cd release_namespace: argo-cd - create_namespace: yes - wait: no + create_namespace: true + wait: false kubeconfig: "{{ kubeconfig.path }}" values: "{{ components.argocd.chart_values }}" diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/k3s.yml b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/k3s.yml index 5ce79ed..9ce7bdb 100644 --- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/k3s.yml +++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/k3s.yml @@ -62,3 +62,13 @@ content: "{{ kubectl_config.stdout }}" mode: 0600 no_log: true + +- name: Add label to node object + kubernetes.core.k8s: + name: "{{ ansible_facts.nodename }}" + kind: Node + state: patched + definition: + metadata: + labels: + vm_id: "{{ moref_id }}" diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/registry.yml b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/registry.yml index e9a49a2..ec22bb7 100644 --- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/registry.yml +++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/registry.yml @@ -5,8 +5,8 @@ name: harbor chart_ref: /opt/metacluster/helm-charts/harbor release_namespace: harbor - create_namespace: yes - wait: no + create_namespace: true + wait: false kubeconfig: "{{ kubeconfig.path }}" values: "{{ components.harbor.chart_values }}" diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/storage.yml b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/storage.yml index 00b859c..f48cda7 100644 --- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/storage.yml +++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/storage.yml @@ -4,8 +4,8 @@ name: longhorn chart_ref: /opt/metacluster/helm-charts/longhorn release_namespace: longhorn-system - create_namespace: yes - wait: no + create_namespace: true + wait: false kubeconfig: "{{ kubeconfig.path }}" values: "{{ components.longhorn.chart_values }}" diff --git a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/clusterapi.yml b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/clusterapi.yml index 110eb5a..a26223c 100644 --- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/clusterapi.yml +++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/clusterapi.yml @@ -153,7 +153,7 @@ kubernetes.core.k8s: definition: >- {{ clusterctl_newcluster.stdout }} - wait: yes + wait: true kubeconfig: "{{ kubeconfig.path }}" # TODO: move to git repo @@ -193,6 +193,6 @@ kubernetes.core.k8s: src: /opt/metacluster/cluster-api/cni-calico/{{ components.clusterapi.workload.version.calico }}/calico.yaml state: present - wait: yes + wait: true kubeconfig: "{{ capi_kubeconfig.path }}" # TODO: move to git repo diff --git a/ansible/roles/firstboot/files/ansible_payload/upgrade/playbook.yml b/ansible/roles/firstboot/files/ansible_payload/upgrade/playbook.yml index 629a28e..6d731bb 100644 --- a/ansible/roles/firstboot/files/ansible_payload/upgrade/playbook.yml +++ b/ansible/roles/firstboot/files/ansible_payload/upgrade/playbook.yml @@ -14,6 +14,7 @@ - disks - metacluster - workloadcluster + - decommission - tty - cleanup handlers: diff --git a/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/decommission/tasks/k3s.yml b/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/decommission/tasks/k3s.yml new file mode 100644 index 0000000..71d8157 --- /dev/null +++ b/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/decommission/tasks/k3s.yml @@ -0,0 +1,24 @@ +- name: Cordon node + kubernetes.core.k8s_drain: + name: "{{ decom_node }}" + state: cordon + kubeconfig: "{{ kubeconfig.path }}" + +- name: Drain node + kubernetes.core.k8s_drain: + name: "{{ decom_node }}" + state: drain + delete_options: + ignore_daemonsets: true + delete_emptydir_data: true + wait_sleep: 10 + wait_timeout: 0 + kubeconfig: "{{ kubeconfig.path }}" + +- name: Delete node + kubernetes.core.k8s: + name: "{{ decom_node }}" + kind: node + state: absent + wait: true + kubeconfig: "{{ kubeconfig.path }}" diff --git a/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/decommission/tasks/main.yml b/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/decommission/tasks/main.yml new file mode 100644 index 0000000..3193957 --- /dev/null +++ b/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/decommission/tasks/main.yml @@ -0,0 +1,18 @@ +- name: Lookup node name and moref id for decommissioning + ansible.builtin.set_fact: + decom_node: >- + {{ + lookup('kubernetes.core.k8s', kind='Node', kubeconfig=(kubeconfig.path)) | + json_query('[? metadata.name != `' ~ ansible_facts.nodename ~ '`].metadata.name') | + first + }} + decom_vmid: >- + {{ + lookup('kubernetes.core.k8s', kind='Node', kubeconfig=(kubeconfig.path)) | + json_query('[? metadata.name != `' ~ ansible_facts.nodename ~ '`].metadata.labels.vm_id') | + first + }} + +- import_tasks: storage.yml +- import_tasks: k3s.yml +- import_tasks: virtualmachine.yml diff --git a/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/decommission/tasks/storage.yml b/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/decommission/tasks/storage.yml new file mode 100644 index 0000000..00e3e98 --- /dev/null +++ b/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/decommission/tasks/storage.yml @@ -0,0 +1,26 @@ +- name: Disable disk scheduling and evict replicas + kubernetes.core.k8s: + name: "{{ decom_node }}" + namespace: longhorn-system + kind: nodes.longhorn.io + state: patched + definition: | + spec: + allowScheduling: false + evictionRequested: true + kubeconfig: "{{ kubeconfig.path }}" + +- name: Reduce replica amount for each volume + kubernetes.core.k8s: + api_version: longhorn.io/v1beta2 + kind: volume + name: "{{ item.metadata.name }}" + namespace: longhorn-system + state: patched + definition: | + spec: + numberOfReplicas: {{ (lookup('kubernetes.core.k8s', kind='node', kubeconfig=(kubeconfig.path)) | length | int) - 1 }} + kubeconfig: "{{ kubeconfig.path }}" + loop: "{{ lookup('kubernetes.core.k8s', api_version='longhorn.io/v1beta2', kind='volume', namespace='longhorn-system', kubeconfig=(kubeconfig.path)) }}" + loop_control: + label: "{{ item.metadata.name }}" diff --git a/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/decommission/tasks/virtualmachine.yml b/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/decommission/tasks/virtualmachine.yml new file mode 100644 index 0000000..6ce9b18 --- /dev/null +++ b/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/decommission/tasks/virtualmachine.yml @@ -0,0 +1,27 @@ +- block: + + - name: Lookup VM name + community.vmware.vmware_guest_info: + moid: "{{ decom_vmid }}" + register: virtualmachine_details + + - name: Power off VM + community.vmware.vmware_guest: + name: "{{ virtualmachine_details.hw_name }}" + folder: "{{ virtualmachine_details.hw_folder }}" + state: poweredoff + # state_change_timeout: "{{ playbook.delay.long }}" + + # - name: Delete VM + # community.vmware.vmware_guest: + # name: "{{ virtualmachine_details.hw_name }}" + # folder: "{{ virtualmachine_details.hw_folder }}" + # state: absent + + module_defaults: + group/vmware: + hostname: "{{ vapp['hv.fqdn'] }}" + validate_certs: no + username: "{{ vapp['hv.username'] }}" + password: "{{ vapp['hv.password'] }}" + datacenter: "{{ vcenter_info.datacenter }}" diff --git a/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/init.yml b/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/init.yml index e7a0f1f..521a1df 100644 --- a/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/init.yml +++ b/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/init.yml @@ -28,3 +28,8 @@ - name: Update certificate truststore ansible.builtin.command: cmd: update-ca-certificates + +- name: Remove redundant files + ansible.builtin.file: + path: /var/lib/rancher/k3s/server/manifests/traefik-config.yaml + state: absent diff --git a/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/k3s.yml b/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/k3s.yml index 4a6bb5a..8e8f26c 100644 --- a/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/k3s.yml +++ b/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/k3s.yml @@ -62,3 +62,13 @@ content: "{{ kubectl_config.stdout }}" mode: 0600 no_log: true + +- name: Add label to node object + kubernetes.core.k8s: + name: "{{ ansible_facts.nodename }}" + kind: Node + state: patched + definition: + metadata: + labels: + vm_id: "{{ moref_id }}" diff --git a/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/registry.yml b/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/registry.yml index cf8f708..f23af6e 100644 --- a/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/registry.yml +++ b/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/registry.yml @@ -5,8 +5,8 @@ name: harbor chart_ref: /opt/metacluster/helm-charts/harbor release_namespace: harbor - create_namespace: yes - wait: no + create_namespace: true + wait: false kubeconfig: "{{ kubeconfig.path }}" values: "{{ components.harbor.chart_values }}" diff --git a/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/storage.yml b/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/storage.yml index cb0c321..70c99a9 100644 --- a/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/storage.yml +++ b/ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/storage.yml @@ -27,13 +27,13 @@ retries: "{{ playbook.retries }}" delay: "{{ playbook.delay.long }}" - - name: Install longhorn chart + - name: Upgrade longhorn chart kubernetes.core.helm: name: longhorn chart_ref: /opt/metacluster/helm-charts/longhorn release_namespace: longhorn-system - create_namespace: yes - wait: no + create_namespace: true + wait: false kubeconfig: "{{ kubeconfig.path }}" values: "{{ components.longhorn.chart_values }}" diff --git a/ansible/vars/metacluster.yml b/ansible/vars/metacluster.yml index 914de93..6191bf5 100644 --- a/ansible/vars/metacluster.yml +++ b/ansible/vars/metacluster.yml @@ -54,7 +54,7 @@ components: argo-cd: helm: - version: 5.14.1 # (= ArgoCD v2.5.2) + version: 5.19.14 # (= ArgoCD v2.5.10) chart: argo/argo-cd parse_logic: helm template . | yq --no-doc eval '.. | .image? | select(.)' | sort -u | awk '!/ /' chart_values: !unsafe | @@ -98,7 +98,7 @@ components: gitea: helm: - version: v6.0.3 # (= Gitea v1.17.3) + version: v7.0.2 # (= Gitea v1.18.3) chart: gitea-charts/gitea parse_logic: helm template . | yq --no-doc eval '.. | .image? | select(.)' | sort -u | sed '/:/!s/$/:latest/' chart_values: !unsafe | diff --git a/packer/vsphere.auto.pkrvars.hcl b/packer/vsphere.auto.pkrvars.hcl index 5329ad1..a374df8 100644 --- a/packer/vsphere.auto.pkrvars.hcl +++ b/packer/vsphere.auto.pkrvars.hcl @@ -2,8 +2,8 @@ vcenter_server = "bv11-vc.bessems.lan" vsphere_username = "administrator@vsphere.local" vsphere_datacenter = "DeSchakel" vsphere_cluster = "Cluster.01" -vsphere_host = "bv11-esx01.bessems.lan" -vsphere_datastore = "ESX01.SSD02" +vsphere_host = "bv11-esx02.bessems.lan" +vsphere_datastore = "ESX02.SSD02" vsphere_folder = "/Packer" vsphere_templatefolder = "/Templates" vsphere_network = "LAN"