From d343b84b30270fa6267a2cb34382600076f9b906 Mon Sep 17 00:00:00 2001
From: Danny Bessems <danny@bessems.eu>
Date: Wed, 4 Jan 2023 13:22:35 +0100
Subject: [PATCH] Add preflight check;Refactor readychecks;Quote input
 variables;Fix kustomization template;Apply kustomization;Generate new
 cluster-api manifest

---
 .../files/ansible_payload/playbook.yml        |   1 +
 .../roles/metacluster/tasks/certauthority.yml | 247 ++++++++++--------
 .../roles/metacluster/tasks/git.yml           |   6 +-
 .../roles/metacluster/tasks/gitops.yml        |   7 +-
 .../roles/metacluster/tasks/registry.yml      |   4 +-
 .../roles/metacluster/tasks/storage.yml       |   2 +-
 .../roles/preflight/tasks/main.yml            |   7 +
 .../workloadcluster/tasks/clusterapi.yml      |  31 +++
 .../workloadcluster/tasks/hypervisor.yml      |  10 +-
 .../workloadcluster/tasks/nodetemplates.yml   |  12 +-
 .../kustomization.cluster-template.j2         |  10 +-
 11 files changed, 201 insertions(+), 136 deletions(-)
 create mode 100644 ansible/roles/firstboot/files/ansible_payload/roles/preflight/tasks/main.yml

diff --git a/ansible/roles/firstboot/files/ansible_payload/playbook.yml b/ansible/roles/firstboot/files/ansible_payload/playbook.yml
index 1ab6933..b7ff5b8 100644
--- a/ansible/roles/firstboot/files/ansible_payload/playbook.yml
+++ b/ansible/roles/firstboot/files/ansible_payload/playbook.yml
@@ -8,6 +8,7 @@
   roles:
     - vapp
     - network
+    - preflight
     - users
     - disks
     - metacluster
diff --git a/ansible/roles/firstboot/files/ansible_payload/roles/metacluster/tasks/certauthority.yml b/ansible/roles/firstboot/files/ansible_payload/roles/metacluster/tasks/certauthority.yml
index 705834c..354e42c 100644
--- a/ansible/roles/firstboot/files/ansible_payload/roles/metacluster/tasks/certauthority.yml
+++ b/ansible/roles/firstboot/files/ansible_payload/roles/metacluster/tasks/certauthority.yml
@@ -1,122 +1,141 @@
-- name: Install step-ca chart
-  kubernetes.core.helm:
-    name: step-certificates
-    chart_ref: /opt/metacluster/helm-charts/step-certificates
-    release_namespace: step-ca
-    create_namespace: yes
-    wait: yes
-    kubeconfig: "{{ kubeconfig.path }}"
-    values: "{{ components.stepcertificates.chart_values }}"
+- block:
 
-- name: Retrieve configmap w/ root certificate
-  kubernetes.core.k8s_info:
-    kind: ConfigMap
-    name: step-certificates-certs
-    namespace: step-ca
-    kubeconfig: "{{ kubeconfig.path }}"
-  register: stepca_cm_certs
+    - name: Install step-ca chart
+      kubernetes.core.helm:
+        name: step-certificates
+        chart_ref: /opt/metacluster/helm-charts/step-certificates
+        release_namespace: step-ca
+        create_namespace: yes
+        wait: no
+        kubeconfig: "{{ kubeconfig.path }}"
+        values: "{{ components.stepcertificates.chart_values }}"
 
-- name: Create target namespaces
-  kubernetes.core.k8s:
-    kind: Namespace
-    name: "{{ item }}"
-    state: present
-    kubeconfig: "{{ kubeconfig.path }}"
-  loop:
-    - argo-cd
-    # - kube-system
+    - name: Ensure step-ca API availability
+      ansible.builtin.uri:
+        url: https://ca.{{ vapp['metacluster.fqdn'] }}/health
+        method: GET
+      register: api_readycheck
+      until:
+        - api_readycheck.json.status is defined
+        - api_readycheck.json.status == 'ok'
+      retries: 5
+      delay: 60
 
-- name: Store root certificate in namespaced configmaps/secrets
-  kubernetes.core.k8s:
-    state: present
-    template: "{{ item.kind }}.j2"
-    kubeconfig: "{{ kubeconfig.path }}"
-  vars:
-    _template:
-      name: "{{ item.name }}"
-      namespace: "{{ item.namespace }}"
-      annotations: "{{ item.annotations | default('{}') | indent(width=4, first=True) }}"
-      labels: "{{ item.labels | default('{}') | indent(width=4, first=True) }}"
-      data: "{{ item.data }}"
-  loop:
-    - name: argocd-tls-certs-cm
-      namespace: argo-cd
-      kind: configmap
-      annotations: |
-        meta.helm.sh/release-name: argo-cd
-        meta.helm.sh/release-namespace: argo-cd
-      labels: |
-        app.kubernetes.io/managed-by: Helm
-        app.kubernetes.io/name: argocd-cm
-        app.kubernetes.io/part-of: argocd
-      data:
-        - key: git.{{ vapp['metacluster.fqdn'] }}
-          value: "{{ stepca_cm_certs.resources[0].data['root_ca.crt'] }}"
-    - name: step-certificates-certs
-      namespace: kube-system
-      kind: secret
-      data:
-        - key: root_ca.crt
-          value: "{{ stepca_cm_certs.resources[0].data['root_ca.crt'] | b64encode }}"
-  loop_control:
-    label: "{{ item.kind + '/' + item.name + ' (' + item.namespace + ')' }}"
+    - name: Retrieve configmap w/ root certificate
+      kubernetes.core.k8s_info:
+        kind: ConfigMap
+        name: step-certificates-certs
+        namespace: step-ca
+        kubeconfig: "{{ kubeconfig.path }}"
+      register: stepca_cm_certs
 
-- name: Configure step-ca passthrough ingress
-  ansible.builtin.template:
-    src: ingressroutetcp.j2
-    dest: /var/lib/rancher/k3s/server/manifests/{{ _template.name }}-manifest.yaml
-    owner: root
-    group: root
-    mode: 0600
-  vars:
-    _template:
-      name: step-ca
-      namespace: step-ca
-      config: |2
-          entryPoints:
-            - websecure
-          routes:
-          - match: HostSNI(`ca.{{ vapp['metacluster.fqdn'] }}`)
-            services:
-            - name: step-certificates
-              port: 443
-          tls:
-            passthrough: true
-  notify:
-    - Apply manifests
+    - name: Create target namespaces
+      kubernetes.core.k8s:
+        kind: Namespace
+        name: "{{ item }}"
+        state: present
+        kubeconfig: "{{ kubeconfig.path }}"
+      loop:
+        - argo-cd
+        # - kube-system
 
-- name: Inject step-ca certificate into traefik container
-  ansible.builtin.blockinfile:
-    path: /var/lib/rancher/k3s/server/manifests/traefik-config.yaml
-    block: |2
-          volumes:
-            - name: step-certificates-certs
-              mountPath: /step-ca
-              type: secret
-          env:
-            - name: LEGO_CA_CERTIFICATES
-              value: /step-ca/root_ca.crt
-    marker: '    # {mark} ANSIBLE MANAGED BLOCK'
-  notify:
-    - Apply manifests
+    - name: Store root certificate in namespaced configmaps/secrets
+      kubernetes.core.k8s:
+        state: present
+        template: "{{ item.kind }}.j2"
+        kubeconfig: "{{ kubeconfig.path }}"
+      vars:
+        _template:
+          name: "{{ item.name }}"
+          namespace: "{{ item.namespace }}"
+          annotations: "{{ item.annotations | default('{}') | indent(width=4, first=True) }}"
+          labels: "{{ item.labels | default('{}') | indent(width=4, first=True) }}"
+          data: "{{ item.data }}"
+      loop:
+        - name: argocd-tls-certs-cm
+          namespace: argo-cd
+          kind: configmap
+          annotations: |
+            meta.helm.sh/release-name: argo-cd
+            meta.helm.sh/release-namespace: argo-cd
+          labels: |
+            app.kubernetes.io/managed-by: Helm
+            app.kubernetes.io/name: argocd-cm
+            app.kubernetes.io/part-of: argocd
+          data:
+            - key: git.{{ vapp['metacluster.fqdn'] }}
+              value: "{{ stepca_cm_certs.resources[0].data['root_ca.crt'] }}"
+        - name: step-certificates-certs
+          namespace: kube-system
+          kind: secret
+          data:
+            - key: root_ca.crt
+              value: "{{ stepca_cm_certs.resources[0].data['root_ca.crt'] | b64encode }}"
+      loop_control:
+        label: "{{ item.kind + '/' + item.name + ' (' + item.namespace + ')' }}"
 
-- name: Trigger handlers
-  ansible.builtin.meta: flush_handlers
+    - name: Configure step-ca passthrough ingress
+      ansible.builtin.template:
+        src: ingressroutetcp.j2
+        dest: /var/lib/rancher/k3s/server/manifests/{{ _template.name }}-manifest.yaml
+        owner: root
+        group: root
+        mode: 0600
+      vars:
+        _template:
+          name: step-ca
+          namespace: step-ca
+          config: |2
+              entryPoints:
+                - websecure
+              routes:
+              - match: HostSNI(`ca.{{ vapp['metacluster.fqdn'] }}`)
+                services:
+                - name: step-certificates
+                  port: 443
+              tls:
+                passthrough: true
+      notify:
+        - Apply manifests
 
-- name: Retrieve step-ca configuration
-  kubernetes.core.k8s_info:
-    kind: ConfigMap
-    name: step-certificates-config
-    namespace: step-ca
-    kubeconfig: "{{ kubeconfig.path }}"
-  register: stepca_cm_config
+    - name: Inject step-ca certificate into traefik container
+      ansible.builtin.blockinfile:
+        path: /var/lib/rancher/k3s/server/manifests/traefik-config.yaml
+        block: |2
+              volumes:
+                - name: step-certificates-certs
+                  mountPath: /step-ca
+                  type: secret
+              env:
+                - name: LEGO_CA_CERTIFICATES
+                  value: /step-ca/root_ca.crt
+        marker: '    # {mark} ANSIBLE MANAGED BLOCK'
+      notify:
+        - Apply manifests
 
-- name: Install root CA in system truststore
-  ansible.builtin.shell:
-    cmd: >-
-      step ca bootstrap \
-        --ca-url=https://ca.{{ vapp['metacluster.fqdn'] }} \
-        --fingerprint={{ stepca_cm_config.resources[0].data['defaults.json'] | from_json | json_query('fingerprint') }} \
-        --install \
-        --force
-      update-ca-certificates
+    - name: Trigger handlers
+      ansible.builtin.meta: flush_handlers
+
+    - name: Retrieve step-ca configuration
+      kubernetes.core.k8s_info:
+        kind: ConfigMap
+        name: step-certificates-config
+        namespace: step-ca
+        kubeconfig: "{{ kubeconfig.path }}"
+      register: stepca_cm_config
+
+    - name: Install root CA in system truststore
+      ansible.builtin.shell:
+        cmd: >-
+          step ca bootstrap \
+            --ca-url=https://ca.{{ vapp['metacluster.fqdn'] }} \
+            --fingerprint={{ stepca_cm_config.resources[0].data['defaults.json'] | from_json | json_query('fingerprint') }} \
+            --install \
+            --force
+          update-ca-certificates
+
+  module_defaults:
+    ansible.builtin.uri:
+      validate_certs: no
+      status_code: [200, 201]
+      body_format: json
diff --git a/ansible/roles/firstboot/files/ansible_payload/roles/metacluster/tasks/git.yml b/ansible/roles/firstboot/files/ansible_payload/roles/metacluster/tasks/git.yml
index 7d54655..32c72b0 100644
--- a/ansible/roles/firstboot/files/ansible_payload/roles/metacluster/tasks/git.yml
+++ b/ansible/roles/firstboot/files/ansible_payload/roles/metacluster/tasks/git.yml
@@ -15,9 +15,11 @@
         url: https://git.{{ vapp['metacluster.fqdn'] }}/api/healthz
         method: GET
       register: api_readycheck
-      until: api_readycheck.json.status is defined
+      until:
+        - api_readycheck.json.status is defined
+        - api_readycheck.json.status == 'pass'
       retries: 5
-      delay: 30
+      delay: 60
 
     - name: Configure additional SSH ingress
       ansible.builtin.template:
diff --git a/ansible/roles/firstboot/files/ansible_payload/roles/metacluster/tasks/gitops.yml b/ansible/roles/firstboot/files/ansible_payload/roles/metacluster/tasks/gitops.yml
index 3e395c6..9fa3fda 100644
--- a/ansible/roles/firstboot/files/ansible_payload/roles/metacluster/tasks/gitops.yml
+++ b/ansible/roles/firstboot/files/ansible_payload/roles/metacluster/tasks/gitops.yml
@@ -6,7 +6,7 @@
         chart_ref: /opt/metacluster/helm-charts/argo-cd
         release_namespace: argo-cd
         create_namespace: yes
-        wait: yes
+        wait: no
         kubeconfig: "{{ kubeconfig.path }}"
         values: "{{ components.argocd.chart_values }}"
 
@@ -15,9 +15,10 @@
         url: https://gitops.{{ vapp['metacluster.fqdn'] }}/api/version
         method: GET
       register: api_readycheck
-      until: api_readycheck.json.Version is defined
+      until:
+        - api_readycheck.json.Version is defined
       retries: 5
-      delay: 30
+      delay: 60
 
     - name: Generate argo-cd API token
       ansible.builtin.uri:
diff --git a/ansible/roles/firstboot/files/ansible_payload/roles/metacluster/tasks/registry.yml b/ansible/roles/firstboot/files/ansible_payload/roles/metacluster/tasks/registry.yml
index 806dd75..61608bd 100644
--- a/ansible/roles/firstboot/files/ansible_payload/roles/metacluster/tasks/registry.yml
+++ b/ansible/roles/firstboot/files/ansible_payload/roles/metacluster/tasks/registry.yml
@@ -6,7 +6,7 @@
         chart_ref: /opt/metacluster/helm-charts/harbor
         release_namespace: harbor
         create_namespace: yes
-        wait: yes
+        wait: no
         kubeconfig: "{{ kubeconfig.path }}"
         values: "{{ components.harbor.chart_values }}"
 
@@ -19,7 +19,7 @@
         - api_readycheck.json.status is defined
         - api_readycheck.json.status == 'healthy'
       retries: 5
-      delay: 30
+      delay: 60
 
     - name: Push images to registry
       ansible.builtin.shell:
diff --git a/ansible/roles/firstboot/files/ansible_payload/roles/metacluster/tasks/storage.yml b/ansible/roles/firstboot/files/ansible_payload/roles/metacluster/tasks/storage.yml
index 45ab68e..01a1c2a 100644
--- a/ansible/roles/firstboot/files/ansible_payload/roles/metacluster/tasks/storage.yml
+++ b/ansible/roles/firstboot/files/ansible_payload/roles/metacluster/tasks/storage.yml
@@ -17,7 +17,7 @@
       until:
         - api_readycheck is not failed
       retries: 5
-      delay: 30
+      delay: 60
 
   module_defaults:
     ansible.builtin.uri:
diff --git a/ansible/roles/firstboot/files/ansible_payload/roles/preflight/tasks/main.yml b/ansible/roles/firstboot/files/ansible_payload/roles/preflight/tasks/main.yml
new file mode 100644
index 0000000..2dbfa4e
--- /dev/null
+++ b/ansible/roles/firstboot/files/ansible_payload/roles/preflight/tasks/main.yml
@@ -0,0 +1,7 @@
+- name: Check for vCenter connectivity
+  community.vmware.vmware_vcenter_settings_info:
+    hostname: "{{ vapp['hv.fqdn'] }}"
+    username: "{{ vapp['hv.username'] }}"
+    password: "{{ vapp['hv.password'] }}"
+    schema: vsphere
+  register: vcenter_info
diff --git a/ansible/roles/firstboot/files/ansible_payload/roles/workloadcluster/tasks/clusterapi.yml b/ansible/roles/firstboot/files/ansible_payload/roles/workloadcluster/tasks/clusterapi.yml
index 8d46462..5d92e33 100644
--- a/ansible/roles/firstboot/files/ansible_payload/roles/workloadcluster/tasks/clusterapi.yml
+++ b/ansible/roles/firstboot/files/ansible_payload/roles/workloadcluster/tasks/clusterapi.yml
@@ -55,6 +55,11 @@
         - update-ca-certificates
         - bash /root/network.sh
 
+- name: Store custom cluster-template
+  ansible.builtin.copy:
+    dest: /opt/metacluster/cluster-api/custom-cluster-template.yaml
+    content: "{{ lookup('kubernetes.core.kustomize', dir='/opt/metacluster/cluster-api/infrastructure-vsphere/' + {{ components.clusterapi.management.version.infrastructure_vsphere }}) }}"
+
 - name: Initialize Cluster API management cluster
   ansible.builtin.shell:
     cmd: >-
@@ -65,3 +70,29 @@
         --config ./clusterctl.yaml \
         --kubeconfig {{ kubeconfig.path }}
     chdir: /opt/metacluster/cluster-api
+
+- name: Parse vApp for workload cluster sizing
+  ansible.builtin.set_fact:
+    clustersize: >-
+      {{ {
+        'controlplane': vapp['deployment.type'] | regex_findall('^cp(\d)+') | first,
+        'workers': vapp['deployment.type'] | regex_findall('w(\d)+$') | first
+      } }}
+
+- name: Generate workload cluster manifest
+  ansible.builtin.shell:
+    cmd: >-
+      clusterctl generate cluster \
+        vapp['workloadcluster.name'] \
+        --control-plane-machine-count {{ clustersize.controlplane }} \
+        --worker-machine-count {{ clustersize.workers }} \
+        --from ./custom-cluster-template.yaml \
+        --config ./clusterctl.yaml \
+        --kubeconfig {{ kubeconfig.path }}
+    chdir: /opt/metacluster/cluster-api
+  register: clusterctl_newcluster
+
+- name: Save workload cluster manifest
+  ansible.builtin.copy:
+    dest: /opt/metacluster/cluster-api/new-cluster.yaml
+    content: "{{ clusterctl_newcluster.stdout }}"
diff --git a/ansible/roles/firstboot/files/ansible_payload/roles/workloadcluster/tasks/hypervisor.yml b/ansible/roles/firstboot/files/ansible_payload/roles/workloadcluster/tasks/hypervisor.yml
index 41eaa67..5406584 100644
--- a/ansible/roles/firstboot/files/ansible_payload/roles/workloadcluster/tasks/hypervisor.yml
+++ b/ansible/roles/firstboot/files/ansible_payload/roles/workloadcluster/tasks/hypervisor.yml
@@ -60,12 +60,12 @@
   ansible.builtin.shell:
     cmd: >-
       npp-prepper \
-        --server {{ vapp['hv.fqdn'] }} \
-        --username {{ vapp['hv.username'] }} \
-        --password {{ vapp['hv.password'] }} \
+        --server "{{ vapp['hv.fqdn'] }}" \
+        --username "{{ vapp['hv.username'] }}" \
+        --password "{{ vapp['hv.password'] }}" \
         dc \
-        --name {{ vcenter_info.datacenter }} \
-        --portgroup {{ vcenter_info.network }} \
+        --name "{{ vcenter_info.datacenter }}" \
+        --portgroup "{{ vcenter_info.network }}" \
         --startaddress {{ vapp['ippool.startip'] }} \
         --endaddress {{ vapp['ippool.endip'] }} \
         --netmask {{ (vapp['guestinfo.ipaddress'] + '/' + vapp['guestinfo.prefixlength']) | ansible.utils.ipaddr('netmask') }} \
diff --git a/ansible/roles/firstboot/files/ansible_payload/roles/workloadcluster/tasks/nodetemplates.yml b/ansible/roles/firstboot/files/ansible_payload/roles/workloadcluster/tasks/nodetemplates.yml
index 658b6d7..2060474 100644
--- a/ansible/roles/firstboot/files/ansible_payload/roles/workloadcluster/tasks/nodetemplates.yml
+++ b/ansible/roles/firstboot/files/ansible_payload/roles/workloadcluster/tasks/nodetemplates.yml
@@ -41,13 +41,13 @@
       ansible.builtin.shell:
         cmd: >-
           npp-prepper \
-            --server {{ vapp['hv.fqdn'] }} \
-            --username {{ vapp['hv.username'] }} \
-            --password {{ vapp['hv.password'] }} \
+            --server "{{ vapp['hv.fqdn'] }}" \
+            --username "{{ vapp['hv.username'] }}" \
+            --password "{{ vapp['hv.password'] }}" \
             vm \
-            --datacenter {{ vcenter_info.datacenter }} \
-            --portgroup {{ vcenter_info.network }} \
-            --name {{ item.instance.hw_name }}
+            --datacenter "{{ vcenter_info.datacenter }}" \
+            --portgroup "{{ vcenter_info.network }}" \
+            --name "{{ item.instance.hw_name }}"
       when: existing_ova.results[index] is failed
       loop: "{{ ova_deploy.results }}"
       loop_control:
diff --git a/ansible/roles/firstboot/files/ansible_payload/templates/kustomization.cluster-template.j2 b/ansible/roles/firstboot/files/ansible_payload/templates/kustomization.cluster-template.j2
index 09e382d..71e74a5 100644
--- a/ansible/roles/firstboot/files/ansible_payload/templates/kustomization.cluster-template.j2
+++ b/ansible/roles/firstboot/files/ansible_payload/templates/kustomization.cluster-template.j2
@@ -39,12 +39,14 @@ patchesStrategicMerge:
             content: |
               {{ _template.script.encoded }}
             permissions: '0744'
+            owner: root:root
+            path: /root/network.sh
           - content: |
               network: {config: disabled}
             owner: root:root
             path: /etc/cloud/cloud.cfg.d/99-disable-network-config.cfg
           - content: |
-              {{ _template.rootca | indent(width=6, first=True) }}
+              {{ _template.rootca | indent(width=14, first=False) | trim }}
             owner: root:root
             path: /usr/local/share/ca-certificates/root_ca.crt
 
@@ -61,6 +63,8 @@ patchesJson6902:
           encoding: base64
           content: |
             {{ _template.script.encoded }}
+          owner: root:root
+          path: /root/network.sh
           permissions: '0744'
       - op: add
         path: /spec/kubeadmConfigSpec/files/-
@@ -73,7 +77,7 @@ patchesJson6902:
         path: /spec/kubeadmConfigSpec/files/-
         value:
           content: |
-            {{ _template.rootca | indent(width=8, first=True) }}
+            {{ _template.rootca | indent(width=12, first=False) | trim }}
           owner: root:root
           path: /usr/local/share/ca-certificates/root_ca.crt
   - target:
@@ -95,6 +99,6 @@ patchesJson6902:
     patch: |-
 {% for cmd in _template.runcmds %}
       - op: add
-        path: /spec/template/spec/preKubeadmCommands/-
+        path: /spec/kubeadmConfigSpec/preKubeadmCommands/-
         value: {{ cmd }}
 {% endfor %}