Packer.Images/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/clusterapi.yml

- block:
  # Below tasks circumvent usernames with `<domain>\<username>` format, which causes CAPV to
  # incorrectly interpret the backslash (despite automatic escaping) as an escape sequence.
  # `vcenter_session.user` will instead contain the username in `<username>@<domain>` format.

    - name: Generate vCenter API token
      ansible.builtin.uri:
        url: https://{{ vapp['hv.fqdn'] }}/api/session
        method: POST
        headers:
          Authorization: Basic {{ ( vapp['hv.username'] ~ ':' ~ vapp['hv.password'] ) | b64encode }}
      register: vcenterapi_token

    - name: Retrieve vCenter API session details
      ansible.builtin.uri:
        url: https://{{ vapp['hv.fqdn'] }}/api/session
        method: GET
        headers:
          vmware-api-session-id: "{{ vcenterapi_token.json }}"
      register: vcenter_session

  module_defaults:
    ansible.builtin.uri:
      validate_certs: no
      status_code: [200, 201]
      body_format: json

- name: Configure clusterctl
  ansible.builtin.template:
    src: clusterctl.j2
    dest: /opt/metacluster/cluster-api/clusterctl.yaml
  vars:
    _template:
      version:
        base: "{{ components.clusterapi.management.version.base }}"
        cert_manager: "{{ components.clusterapi.management.version.cert_manager }}"
        infrastructure_vsphere: "{{ components.clusterapi.management.version.infrastructure_vsphere }}"
        ipam_incluster: "{{ components.clusterapi.management.version.ipam_incluster }}"
      hv:
        fqdn: "{{ vapp['hv.fqdn'] }}"
        tlsthumbprint: "{{ tls_thumbprint.stdout }}"
        username: "{{ vcenter_session.json.user }}"
        password: "{{ vapp['hv.password'] }}"
        datacenter: "{{ vcenter_info.datacenter }}"
        datastore: "{{ vcenter_info.datastore }}"
        network: "{{ vcenter_info.network }}"
        resourcepool: "{{ vcenter_info.resourcepool }}"
        folder: "{{ vcenter_info.folder }}"
      cluster:
        nodetemplate: "{{ nodetemplate_inventorypath }}"
        publickey: "{{ vapp['guestinfo.rootsshkey'] }}"
        version: "{{ components.clusterapi.workload.version.k8s }}"
        vip: "{{ vapp['workloadcluster.vip'] }}"

- name: Generate cluster-template kustomization manifest
  ansible.builtin.template:
    src: kustomization.cluster-template.j2
    dest: /opt/metacluster/cluster-api/infrastructure-vsphere/{{ components.clusterapi.management.version.infrastructure_vsphere }}/kustomization.yaml
  vars:
    _template:
      network:
        fqdn: "{{ vapp['metacluster.fqdn'] }}"
        dnsserver: "{{ vapp['guestinfo.dnsserver'] }}"
      nodesize:
        cpu: "{{ config.clusterapi.size_matrix[ vapp['workloadcluster.nodesize'] ].cpu }}"
        memory: "{{ config.clusterapi.size_matrix[ vapp['workloadcluster.nodesize'] ].memory }}"
      rootca: "{{ stepca_cm_certs.resources[0].data['root_ca.crt'] }}"
      runcmds:
        - update-ca-certificates
      registries: "{{ source_registries }}"

- name: Store custom cluster-template
  ansible.builtin.copy:
    dest: /opt/metacluster/cluster-api/custom-cluster-template.yaml
    content: "{{ lookup('kubernetes.core.kustomize', dir='/opt/metacluster/cluster-api/infrastructure-vsphere/' ~ components.clusterapi.management.version.infrastructure_vsphere ) }}"

- name: Initialize Cluster API management cluster
  ansible.builtin.shell:
    cmd: >-
      clusterctl init \
        -v5 \
        --infrastructure vsphere:{{ components.clusterapi.management.version.infrastructure_vsphere }} \
        --ipam in-cluster:{{ components.clusterapi.management.version.ipam_incluster }} \
        --config ./clusterctl.yaml \
        --kubeconfig {{ kubeconfig.path }}      
    chdir: /opt/metacluster/cluster-api

- name: Initialize tempfolder
  ansible.builtin.tempfile:
    state: directory
  register: capi_clustermanifest

- name: Pull existing repository
  ansible.builtin.git:
    repo: https://git.{{ vapp['metacluster.fqdn'] }}/mc/GitOps.ClusterAPI.git
    dest: "{{ capi_clustermanifest.path }}"
    version: main

- name: Generate Cluster API provider manifests
  ansible.builtin.shell:
    cmd: >-
      clusterctl generate provider \
        -v5 \
        --{{ item.type }} {{ item.name }}:{{ item.version }} \
        --config ./clusterctl.yaml > {{ capi_clustermanifest.path }}/provider-{{ item.name }}.yaml      
    chdir: /opt/metacluster/cluster-api
  loop:
    - type: infrastructure
      name: vsphere
      version: "{{ components.clusterapi.management.version.infrastructure_vsphere }}"
    - type: ipam
      name: in-cluster
      version: "{{ components.clusterapi.management.version.ipam_incluster }}"

- name: Split cluster API provider manifests into separate files
  ansible.builtin.shell:
    cmd: >-
      awk 'BEGINFILE {print "---"}{print}' {{ capi_clustermanifest.path }}/provider-*.yaml |
      kubectl slice \
        -o {{ capi_clustermanifest.path }}/providers      

- name: Ensure controller availability
  kubernetes.core.k8s_info:
    kind: Deployment
    name: "{{ item.name }}"
    namespace: "{{ item.namespace }}"
    wait: true
    kubeconfig: "{{ kubeconfig.path }}"
  loop:
    - name: caip-in-cluster-controller-manager
      namespace: caip-in-cluster-system
    - name: capi-controller-manager
      namespace: capi-system
    - name: capv-controller-manager
      namespace: capv-system
  loop_control:
    label: "{{ item.name }}"

- name: Parse vApp for workload cluster sizing
  ansible.builtin.set_fact:
    clustersize: >-
      {{ {
        'controlplane': vapp['deployment.type'] | regex_findall('^cp(\d)+') | first,
        'worker': vapp['deployment.type'] | regex_findall('w(\d)+') | first,
        'workerstorage': vapp['deployment.type'] | regex_findall('ws(\d)+$') | first
      } }}      

- name: Generate workload cluster manifest
  ansible.builtin.shell:
    cmd: >-
      clusterctl generate cluster \
        {{ vapp['workloadcluster.name'] | lower }} \
        --control-plane-machine-count {{ clustersize.controlplane }} \
        --worker-machine-count {{ clustersize.worker }} \
        --from ./custom-cluster-template.yaml \
        --config ./clusterctl.yaml \
        --kubeconfig {{ kubeconfig.path }}      
    chdir: /opt/metacluster/cluster-api
  register: clusterctl_newcluster

- name: Save workload cluster manifest
  ansible.builtin.copy:
    dest: "{{ capi_clustermanifest.path }}/new-cluster.yaml"
    content: "{{ clusterctl_newcluster.stdout }}"

- name: Split workload cluster manifest into separate files
  ansible.builtin.shell:
    cmd: >-
      kubectl slice \
        -f {{ capi_clustermanifest.path }}/new-cluster.yaml \
        -o {{ capi_clustermanifest.path }}/downstream-cluster      

- name: Generate nodepool kustomization manifest
  ansible.builtin.template:
    src: kustomization.nodepool.j2
    dest: "{{ capi_clustermanifest.path }}/kustomization.yaml"
  vars:
    _template:
      cluster:
        name: "{{ vapp['workloadcluster.name'] }}"
      nodepool:
        size: "{{ clustersize.workerstorage }}"
        additionaldisk: "{{ vapp['workloadcluster.additionaldisk'] }}"

- name: Store nodepool manifest
  ansible.builtin.copy:
    dest: "{{ capi_clustermanifest.path }}/nodepool-worker-storage.yaml"
    content: "{{ lookup('kubernetes.core.kustomize', dir=capi_clustermanifest.path) }}"

- name: Split nodepool manifest into separate files
  ansible.builtin.shell:
    cmd: >-
      kubectl slice \
        -f {{ capi_clustermanifest.path }}/nodepool-worker-storage.yaml \
        -o {{ capi_clustermanifest.path }}/downstream-cluster      

- name: Create in-cluster IpPool
  ansible.builtin.template:
    src: ippool.j2
    dest: "{{ capi_clustermanifest.path }}/downstream-cluster/inclusterippool-{{ _template.cluster.name }}.yml"
  vars:
    _template:
      cluster:
        name: "{{ vapp['workloadcluster.name'] | lower }}"
        namespace: default
        network:
          startip: "{{ vapp['ippool.startip'] }}"
          endip: "{{ vapp['ippool.endip'] }}"
          prefix: "{{ vapp['guestinfo.prefixlength'] }}"
          gateway: "{{ vapp['guestinfo.gateway'] }}"

- name: Push git repository
  lvrfrc87.git_acp.git_acp:
    path: "{{ capi_clustermanifest.path }}"
    branch: main
    comment: "Upload manifests"
    add:
      - ./downstream-cluster
      - ./providers
    clean: untracked
    url: https://administrator:{{ vapp['metacluster.password'] | urlencode }}@git.{{ vapp['metacluster.fqdn'] }}/mc/GitOps.ClusterAPI.git
  environment:
    GIT_AUTHOR_NAME: administrator
    GIT_AUTHOR_EMAIL: administrator@{{ vapp['metacluster.fqdn'] }}
    GIT_COMMITTER_NAME: administrator
    GIT_COMMITTER_EMAIL: administrator@{{ vapp['metacluster.fqdn'] }}

# - name: Cleanup tempfolder
#   ansible.builtin.file:
#     path: "{{ capi_clustermanifest.path }}"
#     state: absent
#   when: capi_clustermanifest.path is defined

- name: Configure Cluster API repository
  ansible.builtin.template:
    src: gitrepo.j2
    dest: /var/lib/rancher/k3s/server/manifests/{{ _template.name }}-manifest.yaml
    owner: root
    group: root
    mode: 0600
  vars:
    _template:
      name: gitrepo-mc-gitopsclusterapi
      namespace: argo-cd
      url: https://git.{{ vapp['metacluster.fqdn'] }}/mc/GitOps.ClusterAPI.git
  notify:
    - Apply manifests

- name: WORKAROUND - Wait for ingress ACME requests to complete
  ansible.builtin.shell:
    cmd: >-
      openssl s_client -connect registry.{{ vapp['metacluster.fqdn'] }}:443 -servername registry.{{ vapp['metacluster.fqdn'] }} 2>/dev/null </dev/null | \
        openssl x509 -noout -subject | \
      grep 'subject=CN = registry.{{ vapp['metacluster.fqdn'] }}'      
  register: certificate_subject
  until: certificate_subject is not failed
  retries: "{{ playbook.retries }}"
  delay: "{{ (storage_benchmark | int) * (playbook.delay.medium | int) }}"

- name: Create application
  ansible.builtin.template:
    src: application.j2
    dest: /var/lib/rancher/k3s/server/manifests/{{ _template.application.name }}-manifest.yaml
    owner: root
    group: root
    mode: 0600
  vars:
    _template:
      application:
        name: application-clusterapi-workloadcluster
        namespace: argo-cd
      cluster:
        name: https://kubernetes.default.svc
        namespace: default
      repository:
        url: https://git.{{ vapp['metacluster.fqdn'] }}/mc/GitOps.ClusterAPI.git
        path: downstream-cluster
        revision: main
  notify:
    - Apply manifests

- name: Trigger handlers
  ansible.builtin.meta: flush_handlers

- name: Wait for cluster to be available
  ansible.builtin.shell:
    cmd: >-
      kubectl wait clusters.cluster.x-k8s.io/{{ vapp['workloadcluster.name'] | lower }} \
        --for=condition=Ready \
        --timeout 0s      
  register: cluster_readycheck
  until: cluster_readycheck is succeeded
  retries: "{{ playbook.retries }}"
  delay: "{{ (storage_benchmark | int) * (playbook.delay.long | int) }}"

- name: Initialize tempfile
  ansible.builtin.tempfile:
    state: file
  register: capi_kubeconfig

- name: Retrieve kubeconfig
  ansible.builtin.shell:
    cmd: >-
      clusterctl get kubeconfig \
        {{ vapp['workloadcluster.name'] | lower }} \
        --kubeconfig {{ kubeconfig.path }}      
  register: capi_kubectl_config

- name: Store kubeconfig in tempfile
  ansible.builtin.copy:
    dest: "{{ capi_kubeconfig.path }}"
    content: "{{ capi_kubectl_config.stdout }}"
    mode: 0600
  no_log: true

# TODO: move to git repo
- name: Apply cni plugin manifest
  kubernetes.core.k8s:
    definition: |
      {{
        lookup('ansible.builtin.file', '/opt/metacluster/cluster-api/cni-calico/' ~ components.clusterapi.workload.version.calico ~ '/calico.yaml') |
          regex_replace('# - name: CALICO_IPV4POOL_CIDR', '- name: CALICO_IPV4POOL_CIDR') |
          regex_replace('#   value: "192.168.0.0/16"',    '  value: "172.30.0.0/16"')
      }}      
    state: present
    wait: true
    kubeconfig: "{{ capi_kubeconfig.path }}"
# TODO: move to git repo