Update hypervisor details;Upgrade components;Housekeeping;Add decom tasks;Prevent configuration reset #2;Add morefid label

ansible/roles/firstboot/files/ansible_payload/upgrade/playbook.yml

@@ -14,6 +14,7 @@
     - disks
     - metacluster
     - workloadcluster
+    - decommission
     - tty
     - cleanup
   handlers:

ansible/roles/firstboot/files/ansible_payload/upgrade/roles/decommission/tasks/k3s.yml

@@ -0,0 +1,24 @@
+- name: Cordon node
+  kubernetes.core.k8s_drain:
+    name: "{{ decom_node }}"
+    state: cordon
+    kubeconfig: "{{ kubeconfig.path }}"
+
+- name: Drain node
+  kubernetes.core.k8s_drain:
+    name: "{{ decom_node }}"
+    state: drain
+    delete_options:
+      ignore_daemonsets: true
+      delete_emptydir_data: true
+      wait_sleep: 10
+      wait_timeout: 0
+    kubeconfig: "{{ kubeconfig.path }}"
+
+- name: Delete node
+  kubernetes.core.k8s:
+    name: "{{ decom_node }}"
+    kind: node
+    state: absent
+    wait: true
+    kubeconfig: "{{ kubeconfig.path }}"

ansible/roles/firstboot/files/ansible_payload/upgrade/roles/decommission/tasks/main.yml

@@ -0,0 +1,18 @@
+- name: Lookup node name and moref id for decommissioning
+  ansible.builtin.set_fact:
+    decom_node: >-
+      {{
+        lookup('kubernetes.core.k8s', kind='Node', kubeconfig=(kubeconfig.path)) |
+          json_query('[? metadata.name != `' ~ ansible_facts.nodename ~ '`].metadata.name') |
+          first
+      }}
+    decom_vmid: >-
+      {{
+        lookup('kubernetes.core.k8s', kind='Node', kubeconfig=(kubeconfig.path)) |
+          json_query('[? metadata.name != `' ~ ansible_facts.nodename ~ '`].metadata.labels.vm_id') |
+          first
+      }}
+
+- import_tasks: storage.yml
+- import_tasks: k3s.yml
+- import_tasks: virtualmachine.yml

ansible/roles/firstboot/files/ansible_payload/upgrade/roles/decommission/tasks/storage.yml

@@ -0,0 +1,26 @@
+- name: Disable disk scheduling and evict replicas
+  kubernetes.core.k8s:
+    name: "{{ decom_node }}"
+    namespace: longhorn-system
+    kind: nodes.longhorn.io
+    state: patched
+    definition: |
+      spec:
+        allowScheduling: false
+        evictionRequested: true
+    kubeconfig: "{{ kubeconfig.path }}"
+
+- name: Reduce replica amount for each volume
+  kubernetes.core.k8s:
+    api_version: longhorn.io/v1beta2
+    kind: volume
+    name: "{{ item.metadata.name }}"
+    namespace: longhorn-system
+    state: patched
+    definition: |
+      spec:
+        numberOfReplicas: {{ (lookup('kubernetes.core.k8s', kind='node', kubeconfig=(kubeconfig.path)) | length | int) - 1 }}
+    kubeconfig: "{{ kubeconfig.path }}"
+  loop: "{{ lookup('kubernetes.core.k8s', api_version='longhorn.io/v1beta2', kind='volume', namespace='longhorn-system', kubeconfig=(kubeconfig.path)) }}"
+  loop_control:
+    label: "{{ item.metadata.name }}"

ansible/roles/firstboot/files/ansible_payload/upgrade/roles/decommission/tasks/virtualmachine.yml

@@ -0,0 +1,27 @@
+- block:
+
+    - name: Lookup VM name
+      community.vmware.vmware_guest_info:
+        moid: "{{ decom_vmid }}"
+      register: virtualmachine_details
+
+    - name: Power off VM
+      community.vmware.vmware_guest:
+        name: "{{ virtualmachine_details.hw_name }}"
+        folder: "{{ virtualmachine_details.hw_folder }}"
+        state: poweredoff
+        # state_change_timeout: "{{ playbook.delay.long }}"
+
+    # - name: Delete VM
+    #   community.vmware.vmware_guest:
+    #     name: "{{ virtualmachine_details.hw_name }}"
+    #     folder: "{{ virtualmachine_details.hw_folder }}"
+    #     state: absent
+
+  module_defaults:
+    group/vmware:
+      hostname: "{{ vapp['hv.fqdn'] }}"
+      validate_certs: no
+      username: "{{ vapp['hv.username'] }}"
+      password: "{{ vapp['hv.password'] }}"
+      datacenter: "{{ vcenter_info.datacenter }}"

ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/init.yml

@@ -28,3 +28,8 @@
 - name: Update certificate truststore
   ansible.builtin.command:
     cmd: update-ca-certificates
+
+- name: Remove redundant files
+  ansible.builtin.file:
+    path: /var/lib/rancher/k3s/server/manifests/traefik-config.yaml
+    state: absent

ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/k3s.yml

@@ -62,3 +62,13 @@
     content: "{{ kubectl_config.stdout }}"
     mode: 0600
   no_log: true
+
+- name: Add label to node object
+  kubernetes.core.k8s:
+    name: "{{ ansible_facts.nodename }}"
+    kind: Node
+    state: patched
+    definition:
+      metadata:
+        labels:
+          vm_id: "{{ moref_id }}"

ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/registry.yml

@@ -5,8 +5,8 @@
         name: harbor
         chart_ref: /opt/metacluster/helm-charts/harbor
         release_namespace: harbor
-        create_namespace: yes
-        wait: no
+        create_namespace: true
+        wait: false
         kubeconfig: "{{ kubeconfig.path }}"
         values: "{{ components.harbor.chart_values }}"
 

ansible/roles/firstboot/files/ansible_payload/upgrade/roles/metacluster/tasks/storage.yml

@@ -27,13 +27,13 @@
       retries: "{{ playbook.retries }}"
       delay: "{{ playbook.delay.long }}"
 
-    - name: Install longhorn chart
+    - name: Upgrade longhorn chart
       kubernetes.core.helm:
         name: longhorn
         chart_ref: /opt/metacluster/helm-charts/longhorn
         release_namespace: longhorn-system
-        create_namespace: yes
-        wait: no
+        create_namespace: true
+        wait: false
         kubeconfig: "{{ kubeconfig.path }}"
         values: "{{ components.longhorn.chart_values }}"