diff --git a/ansible/roles/firstboot/files/ansible_payload/playbook.yml b/ansible/roles/firstboot/files/ansible_payload/playbook.yml index 1ab6933..b7ff5b8 100644 --- a/ansible/roles/firstboot/files/ansible_payload/playbook.yml +++ b/ansible/roles/firstboot/files/ansible_payload/playbook.yml @@ -8,6 +8,7 @@ roles: - vapp - network + - preflight - users - disks - metacluster diff --git a/ansible/roles/firstboot/files/ansible_payload/roles/metacluster/tasks/certauthority.yml b/ansible/roles/firstboot/files/ansible_payload/roles/metacluster/tasks/certauthority.yml index 705834c..354e42c 100644 --- a/ansible/roles/firstboot/files/ansible_payload/roles/metacluster/tasks/certauthority.yml +++ b/ansible/roles/firstboot/files/ansible_payload/roles/metacluster/tasks/certauthority.yml @@ -1,122 +1,141 @@ -- name: Install step-ca chart - kubernetes.core.helm: - name: step-certificates - chart_ref: /opt/metacluster/helm-charts/step-certificates - release_namespace: step-ca - create_namespace: yes - wait: yes - kubeconfig: "{{ kubeconfig.path }}" - values: "{{ components.stepcertificates.chart_values }}" +- block: -- name: Retrieve configmap w/ root certificate - kubernetes.core.k8s_info: - kind: ConfigMap - name: step-certificates-certs - namespace: step-ca - kubeconfig: "{{ kubeconfig.path }}" - register: stepca_cm_certs + - name: Install step-ca chart + kubernetes.core.helm: + name: step-certificates + chart_ref: /opt/metacluster/helm-charts/step-certificates + release_namespace: step-ca + create_namespace: yes + wait: no + kubeconfig: "{{ kubeconfig.path }}" + values: "{{ components.stepcertificates.chart_values }}" -- name: Create target namespaces - kubernetes.core.k8s: - kind: Namespace - name: "{{ item }}" - state: present - kubeconfig: "{{ kubeconfig.path }}" - loop: - - argo-cd - # - kube-system + - name: Ensure step-ca API availability + ansible.builtin.uri: + url: https://ca.{{ vapp['metacluster.fqdn'] }}/health + method: GET + register: api_readycheck + until: + - api_readycheck.json.status is defined + - api_readycheck.json.status == 'ok' + retries: 5 + delay: 60 -- name: Store root certificate in namespaced configmaps/secrets - kubernetes.core.k8s: - state: present - template: "{{ item.kind }}.j2" - kubeconfig: "{{ kubeconfig.path }}" - vars: - _template: - name: "{{ item.name }}" - namespace: "{{ item.namespace }}" - annotations: "{{ item.annotations | default('{}') | indent(width=4, first=True) }}" - labels: "{{ item.labels | default('{}') | indent(width=4, first=True) }}" - data: "{{ item.data }}" - loop: - - name: argocd-tls-certs-cm - namespace: argo-cd - kind: configmap - annotations: | - meta.helm.sh/release-name: argo-cd - meta.helm.sh/release-namespace: argo-cd - labels: | - app.kubernetes.io/managed-by: Helm - app.kubernetes.io/name: argocd-cm - app.kubernetes.io/part-of: argocd - data: - - key: git.{{ vapp['metacluster.fqdn'] }} - value: "{{ stepca_cm_certs.resources[0].data['root_ca.crt'] }}" - - name: step-certificates-certs - namespace: kube-system - kind: secret - data: - - key: root_ca.crt - value: "{{ stepca_cm_certs.resources[0].data['root_ca.crt'] | b64encode }}" - loop_control: - label: "{{ item.kind + '/' + item.name + ' (' + item.namespace + ')' }}" + - name: Retrieve configmap w/ root certificate + kubernetes.core.k8s_info: + kind: ConfigMap + name: step-certificates-certs + namespace: step-ca + kubeconfig: "{{ kubeconfig.path }}" + register: stepca_cm_certs -- name: Configure step-ca passthrough ingress - ansible.builtin.template: - src: ingressroutetcp.j2 - dest: /var/lib/rancher/k3s/server/manifests/{{ _template.name }}-manifest.yaml - owner: root - group: root - mode: 0600 - vars: - _template: - name: step-ca - namespace: step-ca - config: |2 - entryPoints: - - websecure - routes: - - match: HostSNI(`ca.{{ vapp['metacluster.fqdn'] }}`) - services: - - name: step-certificates - port: 443 - tls: - passthrough: true - notify: - - Apply manifests + - name: Create target namespaces + kubernetes.core.k8s: + kind: Namespace + name: "{{ item }}" + state: present + kubeconfig: "{{ kubeconfig.path }}" + loop: + - argo-cd + # - kube-system -- name: Inject step-ca certificate into traefik container - ansible.builtin.blockinfile: - path: /var/lib/rancher/k3s/server/manifests/traefik-config.yaml - block: |2 - volumes: - - name: step-certificates-certs - mountPath: /step-ca - type: secret - env: - - name: LEGO_CA_CERTIFICATES - value: /step-ca/root_ca.crt - marker: ' # {mark} ANSIBLE MANAGED BLOCK' - notify: - - Apply manifests + - name: Store root certificate in namespaced configmaps/secrets + kubernetes.core.k8s: + state: present + template: "{{ item.kind }}.j2" + kubeconfig: "{{ kubeconfig.path }}" + vars: + _template: + name: "{{ item.name }}" + namespace: "{{ item.namespace }}" + annotations: "{{ item.annotations | default('{}') | indent(width=4, first=True) }}" + labels: "{{ item.labels | default('{}') | indent(width=4, first=True) }}" + data: "{{ item.data }}" + loop: + - name: argocd-tls-certs-cm + namespace: argo-cd + kind: configmap + annotations: | + meta.helm.sh/release-name: argo-cd + meta.helm.sh/release-namespace: argo-cd + labels: | + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: argocd-cm + app.kubernetes.io/part-of: argocd + data: + - key: git.{{ vapp['metacluster.fqdn'] }} + value: "{{ stepca_cm_certs.resources[0].data['root_ca.crt'] }}" + - name: step-certificates-certs + namespace: kube-system + kind: secret + data: + - key: root_ca.crt + value: "{{ stepca_cm_certs.resources[0].data['root_ca.crt'] | b64encode }}" + loop_control: + label: "{{ item.kind + '/' + item.name + ' (' + item.namespace + ')' }}" -- name: Trigger handlers - ansible.builtin.meta: flush_handlers + - name: Configure step-ca passthrough ingress + ansible.builtin.template: + src: ingressroutetcp.j2 + dest: /var/lib/rancher/k3s/server/manifests/{{ _template.name }}-manifest.yaml + owner: root + group: root + mode: 0600 + vars: + _template: + name: step-ca + namespace: step-ca + config: |2 + entryPoints: + - websecure + routes: + - match: HostSNI(`ca.{{ vapp['metacluster.fqdn'] }}`) + services: + - name: step-certificates + port: 443 + tls: + passthrough: true + notify: + - Apply manifests -- name: Retrieve step-ca configuration - kubernetes.core.k8s_info: - kind: ConfigMap - name: step-certificates-config - namespace: step-ca - kubeconfig: "{{ kubeconfig.path }}" - register: stepca_cm_config + - name: Inject step-ca certificate into traefik container + ansible.builtin.blockinfile: + path: /var/lib/rancher/k3s/server/manifests/traefik-config.yaml + block: |2 + volumes: + - name: step-certificates-certs + mountPath: /step-ca + type: secret + env: + - name: LEGO_CA_CERTIFICATES + value: /step-ca/root_ca.crt + marker: ' # {mark} ANSIBLE MANAGED BLOCK' + notify: + - Apply manifests -- name: Install root CA in system truststore - ansible.builtin.shell: - cmd: >- - step ca bootstrap \ - --ca-url=https://ca.{{ vapp['metacluster.fqdn'] }} \ - --fingerprint={{ stepca_cm_config.resources[0].data['defaults.json'] | from_json | json_query('fingerprint') }} \ - --install \ - --force - update-ca-certificates + - name: Trigger handlers + ansible.builtin.meta: flush_handlers + + - name: Retrieve step-ca configuration + kubernetes.core.k8s_info: + kind: ConfigMap + name: step-certificates-config + namespace: step-ca + kubeconfig: "{{ kubeconfig.path }}" + register: stepca_cm_config + + - name: Install root CA in system truststore + ansible.builtin.shell: + cmd: >- + step ca bootstrap \ + --ca-url=https://ca.{{ vapp['metacluster.fqdn'] }} \ + --fingerprint={{ stepca_cm_config.resources[0].data['defaults.json'] | from_json | json_query('fingerprint') }} \ + --install \ + --force + update-ca-certificates + + module_defaults: + ansible.builtin.uri: + validate_certs: no + status_code: [200, 201] + body_format: json diff --git a/ansible/roles/firstboot/files/ansible_payload/roles/metacluster/tasks/git.yml b/ansible/roles/firstboot/files/ansible_payload/roles/metacluster/tasks/git.yml index 7d54655..32c72b0 100644 --- a/ansible/roles/firstboot/files/ansible_payload/roles/metacluster/tasks/git.yml +++ b/ansible/roles/firstboot/files/ansible_payload/roles/metacluster/tasks/git.yml @@ -15,9 +15,11 @@ url: https://git.{{ vapp['metacluster.fqdn'] }}/api/healthz method: GET register: api_readycheck - until: api_readycheck.json.status is defined + until: + - api_readycheck.json.status is defined + - api_readycheck.json.status == 'pass' retries: 5 - delay: 30 + delay: 60 - name: Configure additional SSH ingress ansible.builtin.template: diff --git a/ansible/roles/firstboot/files/ansible_payload/roles/metacluster/tasks/gitops.yml b/ansible/roles/firstboot/files/ansible_payload/roles/metacluster/tasks/gitops.yml index 3e395c6..9fa3fda 100644 --- a/ansible/roles/firstboot/files/ansible_payload/roles/metacluster/tasks/gitops.yml +++ b/ansible/roles/firstboot/files/ansible_payload/roles/metacluster/tasks/gitops.yml @@ -6,7 +6,7 @@ chart_ref: /opt/metacluster/helm-charts/argo-cd release_namespace: argo-cd create_namespace: yes - wait: yes + wait: no kubeconfig: "{{ kubeconfig.path }}" values: "{{ components.argocd.chart_values }}" @@ -15,9 +15,10 @@ url: https://gitops.{{ vapp['metacluster.fqdn'] }}/api/version method: GET register: api_readycheck - until: api_readycheck.json.Version is defined + until: + - api_readycheck.json.Version is defined retries: 5 - delay: 30 + delay: 60 - name: Generate argo-cd API token ansible.builtin.uri: diff --git a/ansible/roles/firstboot/files/ansible_payload/roles/metacluster/tasks/registry.yml b/ansible/roles/firstboot/files/ansible_payload/roles/metacluster/tasks/registry.yml index 806dd75..61608bd 100644 --- a/ansible/roles/firstboot/files/ansible_payload/roles/metacluster/tasks/registry.yml +++ b/ansible/roles/firstboot/files/ansible_payload/roles/metacluster/tasks/registry.yml @@ -6,7 +6,7 @@ chart_ref: /opt/metacluster/helm-charts/harbor release_namespace: harbor create_namespace: yes - wait: yes + wait: no kubeconfig: "{{ kubeconfig.path }}" values: "{{ components.harbor.chart_values }}" @@ -19,7 +19,7 @@ - api_readycheck.json.status is defined - api_readycheck.json.status == 'healthy' retries: 5 - delay: 30 + delay: 60 - name: Push images to registry ansible.builtin.shell: diff --git a/ansible/roles/firstboot/files/ansible_payload/roles/metacluster/tasks/storage.yml b/ansible/roles/firstboot/files/ansible_payload/roles/metacluster/tasks/storage.yml index 45ab68e..01a1c2a 100644 --- a/ansible/roles/firstboot/files/ansible_payload/roles/metacluster/tasks/storage.yml +++ b/ansible/roles/firstboot/files/ansible_payload/roles/metacluster/tasks/storage.yml @@ -17,7 +17,7 @@ until: - api_readycheck is not failed retries: 5 - delay: 30 + delay: 60 module_defaults: ansible.builtin.uri: diff --git a/ansible/roles/firstboot/files/ansible_payload/roles/preflight/tasks/main.yml b/ansible/roles/firstboot/files/ansible_payload/roles/preflight/tasks/main.yml new file mode 100644 index 0000000..2dbfa4e --- /dev/null +++ b/ansible/roles/firstboot/files/ansible_payload/roles/preflight/tasks/main.yml @@ -0,0 +1,7 @@ +- name: Check for vCenter connectivity + community.vmware.vmware_vcenter_settings_info: + hostname: "{{ vapp['hv.fqdn'] }}" + username: "{{ vapp['hv.username'] }}" + password: "{{ vapp['hv.password'] }}" + schema: vsphere + register: vcenter_info diff --git a/ansible/roles/firstboot/files/ansible_payload/roles/workloadcluster/tasks/clusterapi.yml b/ansible/roles/firstboot/files/ansible_payload/roles/workloadcluster/tasks/clusterapi.yml index 8d46462..5d92e33 100644 --- a/ansible/roles/firstboot/files/ansible_payload/roles/workloadcluster/tasks/clusterapi.yml +++ b/ansible/roles/firstboot/files/ansible_payload/roles/workloadcluster/tasks/clusterapi.yml @@ -55,6 +55,11 @@ - update-ca-certificates - bash /root/network.sh +- name: Store custom cluster-template + ansible.builtin.copy: + dest: /opt/metacluster/cluster-api/custom-cluster-template.yaml + content: "{{ lookup('kubernetes.core.kustomize', dir='/opt/metacluster/cluster-api/infrastructure-vsphere/' + {{ components.clusterapi.management.version.infrastructure_vsphere }}) }}" + - name: Initialize Cluster API management cluster ansible.builtin.shell: cmd: >- @@ -65,3 +70,29 @@ --config ./clusterctl.yaml \ --kubeconfig {{ kubeconfig.path }} chdir: /opt/metacluster/cluster-api + +- name: Parse vApp for workload cluster sizing + ansible.builtin.set_fact: + clustersize: >- + {{ { + 'controlplane': vapp['deployment.type'] | regex_findall('^cp(\d)+') | first, + 'workers': vapp['deployment.type'] | regex_findall('w(\d)+$') | first + } }} + +- name: Generate workload cluster manifest + ansible.builtin.shell: + cmd: >- + clusterctl generate cluster \ + vapp['workloadcluster.name'] \ + --control-plane-machine-count {{ clustersize.controlplane }} \ + --worker-machine-count {{ clustersize.workers }} \ + --from ./custom-cluster-template.yaml \ + --config ./clusterctl.yaml \ + --kubeconfig {{ kubeconfig.path }} + chdir: /opt/metacluster/cluster-api + register: clusterctl_newcluster + +- name: Save workload cluster manifest + ansible.builtin.copy: + dest: /opt/metacluster/cluster-api/new-cluster.yaml + content: "{{ clusterctl_newcluster.stdout }}" diff --git a/ansible/roles/firstboot/files/ansible_payload/roles/workloadcluster/tasks/hypervisor.yml b/ansible/roles/firstboot/files/ansible_payload/roles/workloadcluster/tasks/hypervisor.yml index 41eaa67..5406584 100644 --- a/ansible/roles/firstboot/files/ansible_payload/roles/workloadcluster/tasks/hypervisor.yml +++ b/ansible/roles/firstboot/files/ansible_payload/roles/workloadcluster/tasks/hypervisor.yml @@ -60,12 +60,12 @@ ansible.builtin.shell: cmd: >- npp-prepper \ - --server {{ vapp['hv.fqdn'] }} \ - --username {{ vapp['hv.username'] }} \ - --password {{ vapp['hv.password'] }} \ + --server "{{ vapp['hv.fqdn'] }}" \ + --username "{{ vapp['hv.username'] }}" \ + --password "{{ vapp['hv.password'] }}" \ dc \ - --name {{ vcenter_info.datacenter }} \ - --portgroup {{ vcenter_info.network }} \ + --name "{{ vcenter_info.datacenter }}" \ + --portgroup "{{ vcenter_info.network }}" \ --startaddress {{ vapp['ippool.startip'] }} \ --endaddress {{ vapp['ippool.endip'] }} \ --netmask {{ (vapp['guestinfo.ipaddress'] + '/' + vapp['guestinfo.prefixlength']) | ansible.utils.ipaddr('netmask') }} \ diff --git a/ansible/roles/firstboot/files/ansible_payload/roles/workloadcluster/tasks/nodetemplates.yml b/ansible/roles/firstboot/files/ansible_payload/roles/workloadcluster/tasks/nodetemplates.yml index 658b6d7..2060474 100644 --- a/ansible/roles/firstboot/files/ansible_payload/roles/workloadcluster/tasks/nodetemplates.yml +++ b/ansible/roles/firstboot/files/ansible_payload/roles/workloadcluster/tasks/nodetemplates.yml @@ -41,13 +41,13 @@ ansible.builtin.shell: cmd: >- npp-prepper \ - --server {{ vapp['hv.fqdn'] }} \ - --username {{ vapp['hv.username'] }} \ - --password {{ vapp['hv.password'] }} \ + --server "{{ vapp['hv.fqdn'] }}" \ + --username "{{ vapp['hv.username'] }}" \ + --password "{{ vapp['hv.password'] }}" \ vm \ - --datacenter {{ vcenter_info.datacenter }} \ - --portgroup {{ vcenter_info.network }} \ - --name {{ item.instance.hw_name }} + --datacenter "{{ vcenter_info.datacenter }}" \ + --portgroup "{{ vcenter_info.network }}" \ + --name "{{ item.instance.hw_name }}" when: existing_ova.results[index] is failed loop: "{{ ova_deploy.results }}" loop_control: diff --git a/ansible/roles/firstboot/files/ansible_payload/templates/kustomization.cluster-template.j2 b/ansible/roles/firstboot/files/ansible_payload/templates/kustomization.cluster-template.j2 index 09e382d..71e74a5 100644 --- a/ansible/roles/firstboot/files/ansible_payload/templates/kustomization.cluster-template.j2 +++ b/ansible/roles/firstboot/files/ansible_payload/templates/kustomization.cluster-template.j2 @@ -39,12 +39,14 @@ patchesStrategicMerge: content: | {{ _template.script.encoded }} permissions: '0744' + owner: root:root + path: /root/network.sh - content: | network: {config: disabled} owner: root:root path: /etc/cloud/cloud.cfg.d/99-disable-network-config.cfg - content: | - {{ _template.rootca | indent(width=6, first=True) }} + {{ _template.rootca | indent(width=14, first=False) | trim }} owner: root:root path: /usr/local/share/ca-certificates/root_ca.crt @@ -61,6 +63,8 @@ patchesJson6902: encoding: base64 content: | {{ _template.script.encoded }} + owner: root:root + path: /root/network.sh permissions: '0744' - op: add path: /spec/kubeadmConfigSpec/files/- @@ -73,7 +77,7 @@ patchesJson6902: path: /spec/kubeadmConfigSpec/files/- value: content: | - {{ _template.rootca | indent(width=8, first=True) }} + {{ _template.rootca | indent(width=12, first=False) | trim }} owner: root:root path: /usr/local/share/ca-certificates/root_ca.crt - target: @@ -95,6 +99,6 @@ patchesJson6902: patch: |- {% for cmd in _template.runcmds %} - op: add - path: /spec/template/spec/preKubeadmCommands/- + path: /spec/kubeadmConfigSpec/preKubeadmCommands/- value: {{ cmd }} {% endfor %}