Switch from Network Protocol Profiles to in-cluster IPAM

ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/clusterapi.yml

@@ -77,12 +77,12 @@
     _template:
       fqdn: "{{ vapp['metacluster.fqdn'] }}"
       rootca: "{{ stepca_cm_certs.resources[0].data['root_ca.crt'] }}"
-      script:
-        # Base64 encoded; to avoid variable substitution when clusterctl parses the cluster-template.yml
-        encoded: IyEvYmluL2Jhc2gKdm10b29sc2QgLS1jbWQgJ2luZm8tZ2V0IGd1ZXN0aW5mby5vdmZFbnYnID4gL3RtcC9vdmZlbnYKCklQQWRkcmVzcz0kKHNlZCAtbiAncy8uKlByb3BlcnR5IG9lOmtleT0iZ3Vlc3RpbmZvLmludGVyZmFjZS4wLmlwLjAuYWRkcmVzcyIgb2U6dmFsdWU9IlwoW14iXSpcKS4qL1wxL3AnIC90bXAvb3ZmZW52KQpTdWJuZXRNYXNrPSQoc2VkIC1uICdzLy4qUHJvcGVydHkgb2U6a2V5PSJndWVzdGluZm8uaW50ZXJmYWNlLjAuaXAuMC5uZXRtYXNrIiBvZTp2YWx1ZT0iXChbXiJdKlwpLiovXDEvcCcgL3RtcC9vdmZlbnYpCkdhdGV3YXk9JChzZWQgLW4gJ3MvLipQcm9wZXJ0eSBvZTprZXk9Imd1ZXN0aW5mby5pbnRlcmZhY2UuMC5yb3V0ZS4wLmdhdGV3YXkiIG9lOnZhbHVlPSJcKFteIl0qXCkuKi9cMS9wJyAvdG1wL292ZmVudikKRE5TPSQoc2VkIC1uICdzLy4qUHJvcGVydHkgb2U6a2V5PSJndWVzdGluZm8uZG5zLnNlcnZlcnMiIG9lOnZhbHVlPSJcKFteIl0qXCkuKi9cMS9wJyAvdG1wL292ZmVudikKTUFDQWRkcmVzcz0kKHNlZCAtbiAncy8uKnZlOkFkYXB0ZXIgdmU6bWFjPSJcKFteIl0qXCkuKi9cMS9wJyAvdG1wL292ZmVudikKCm1hc2syY2lkcigpIHsKICBjPTAKICB4PTAkKCBwcmludGYgJyVvJyAkezEvLy4vIH0gKQoKICB3aGlsZSBbICR4IC1ndCAwIF07IGRvCiAgICBsZXQgYys9JCgoeCUyKSkgJ3g+Pj0xJwogIGRvbmUKCiAgZWNobyAkYwp9CgpQcmVmaXg9JChtYXNrMmNpZHIgJFN1Ym5ldE1hc2spCgpjYXQgPiAvZXRjL25ldHBsYW4vMDEtbmV0Y2ZnLnlhbWwgPDxFT0YKbmV0d29yazoKICB2ZXJzaW9uOiAyCiAgcmVuZGVyZXI6IG5ldHdvcmtkCiAgZXRoZXJuZXRzOgogICAgaWQwOgogICAgICBzZXQtbmFtZTogZXRoMAogICAgICBtYXRjaDoKICAgICAgICBtYWNhZGRyZXNzOiAkTUFDQWRkcmVzcwogICAgICBhZGRyZXNzZXM6CiAgICAgICAgLSAkSVBBZGRyZXNzLyRQcmVmaXgKICAgICAgZ2F0ZXdheTQ6ICRHYXRld2F5CiAgICAgIG5hbWVzZXJ2ZXJzOgogICAgICAgIGFkZHJlc3NlcyA6IFskRE5TXQpFT0YKcm0gL2V0Yy9uZXRwbGFuLzUwKi55YW1sIC1mCgpzdWRvIG5ldHBsYW4gYXBwbHk=
+      # script:
+      #   # Base64 encoded; to avoid variable substitution when clusterctl parses the cluster-template.yml
+      #   encoded: IyEvYmluL2Jhc2gKdm10b29sc2QgLS1jbWQgJ2luZm8tZ2V0IGd1ZXN0aW5mby5vdmZFbnYnID4gL3RtcC9vdmZlbnYKCklQQWRkcmVzcz0kKHNlZCAtbiAncy8uKlByb3BlcnR5IG9lOmtleT0iZ3Vlc3RpbmZvLmludGVyZmFjZS4wLmlwLjAuYWRkcmVzcyIgb2U6dmFsdWU9IlwoW14iXSpcKS4qL1wxL3AnIC90bXAvb3ZmZW52KQpTdWJuZXRNYXNrPSQoc2VkIC1uICdzLy4qUHJvcGVydHkgb2U6a2V5PSJndWVzdGluZm8uaW50ZXJmYWNlLjAuaXAuMC5uZXRtYXNrIiBvZTp2YWx1ZT0iXChbXiJdKlwpLiovXDEvcCcgL3RtcC9vdmZlbnYpCkdhdGV3YXk9JChzZWQgLW4gJ3MvLipQcm9wZXJ0eSBvZTprZXk9Imd1ZXN0aW5mby5pbnRlcmZhY2UuMC5yb3V0ZS4wLmdhdGV3YXkiIG9lOnZhbHVlPSJcKFteIl0qXCkuKi9cMS9wJyAvdG1wL292ZmVudikKRE5TPSQoc2VkIC1uICdzLy4qUHJvcGVydHkgb2U6a2V5PSJndWVzdGluZm8uZG5zLnNlcnZlcnMiIG9lOnZhbHVlPSJcKFteIl0qXCkuKi9cMS9wJyAvdG1wL292ZmVudikKTUFDQWRkcmVzcz0kKHNlZCAtbiAncy8uKnZlOkFkYXB0ZXIgdmU6bWFjPSJcKFteIl0qXCkuKi9cMS9wJyAvdG1wL292ZmVudikKCm1hc2syY2lkcigpIHsKICBjPTAKICB4PTAkKCBwcmludGYgJyVvJyAkezEvLy4vIH0gKQoKICB3aGlsZSBbICR4IC1ndCAwIF07IGRvCiAgICBsZXQgYys9JCgoeCUyKSkgJ3g+Pj0xJwogIGRvbmUKCiAgZWNobyAkYwp9CgpQcmVmaXg9JChtYXNrMmNpZHIgJFN1Ym5ldE1hc2spCgpjYXQgPiAvZXRjL25ldHBsYW4vMDEtbmV0Y2ZnLnlhbWwgPDxFT0YKbmV0d29yazoKICB2ZXJzaW9uOiAyCiAgcmVuZGVyZXI6IG5ldHdvcmtkCiAgZXRoZXJuZXRzOgogICAgaWQwOgogICAgICBzZXQtbmFtZTogZXRoMAogICAgICBtYXRjaDoKICAgICAgICBtYWNhZGRyZXNzOiAkTUFDQWRkcmVzcwogICAgICBhZGRyZXNzZXM6CiAgICAgICAgLSAkSVBBZGRyZXNzLyRQcmVmaXgKICAgICAgZ2F0ZXdheTQ6ICRHYXRld2F5CiAgICAgIG5hbWVzZXJ2ZXJzOgogICAgICAgIGFkZHJlc3NlcyA6IFskRE5TXQpFT0YKcm0gL2V0Yy9uZXRwbGFuLzUwKi55YW1sIC1mCgpzdWRvIG5ldHBsYW4gYXBwbHk=
       runcmds:
         - update-ca-certificates
-        - bash /root/network.sh
+        # - bash /root/network.sh
 
 - name: Store custom cluster-template
   ansible.builtin.copy:
@@ -159,6 +159,21 @@
     state: absent
   when: capi_clustermanifest.path is defined
 
+- name: Create in-cluster IpPool
+  kubernetes.core.k8s:
+    template: ippool.j2
+    state: present
+    kubeconfig: "{{ kubeconfig.path }}"
+  vars:
+    _template:
+      cluster:
+        name: "{{ vapp['workloadcluster.name'] | lower }}"
+        network:
+          startip: "{{ vapp['ippool.startip'] }}"
+          endip: "{{ vapp['ippool.endip'] }}"
+          prefix: "{{ vapp['guestinfo.prefixlength'] }}"
+          gateway: "{{ vapp['guestinfo.gateway'] }}"
+
 - name: WORKAROUND - Wait for ingress ACME requests to complete
   ansible.builtin.shell:
     cmd: >-

ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/workloadcluster/tasks/hypervisor.yml

@@ -56,20 +56,20 @@
   loop_control:
     label: "{{ item.item.attribute }}"
 
-- name: Configure network protocol profile on hypervisor
-  ansible.builtin.shell:
-    cmd: >-
-      npp-prepper \
-        --server "{{ vapp['hv.fqdn'] }}" \
-        --username "{{ vapp['hv.username'] }}" \
-        --password "{{ vapp['hv.password'] }}" \
-        dc \
-        --name "{{ vcenter_info.datacenter }}" \
-        --portgroup "{{ vcenter_info.network }}" \
-        --startaddress {{ vapp['ippool.startip'] }} \
-        --endaddress {{ vapp['ippool.endip'] }} \
-        --netmask {{ (vapp['guestinfo.ipaddress'] ~ '/' ~ vapp['guestinfo.prefixlength']) | ansible.utils.ipaddr('netmask') }} \
-        {{ vapp['guestinfo.dnsserver'] | split(',') | map('trim') | map('regex_replace', '^', '--dnsserver ') | join(' ') }} \
-        --dnsdomain {{ vapp['metacluster.fqdn'] }} \
-        --gateway {{ vapp['guestinfo.gateway'] }} \
-        --force
+# - name: Configure network protocol profile on hypervisor
+#   ansible.builtin.shell:
+#     cmd: >-
+#       npp-prepper \
+#         --server "{{ vapp['hv.fqdn'] }}" \
+#         --username "{{ vapp['hv.username'] }}" \
+#         --password "{{ vapp['hv.password'] }}" \
+#         dc \
+#         --name "{{ vcenter_info.datacenter }}" \
+#         --portgroup "{{ vcenter_info.network }}" \
+#         --startaddress {{ vapp['ippool.startip'] }} \
+#         --endaddress {{ vapp['ippool.endip'] }} \
+#         --netmask {{ (vapp['guestinfo.ipaddress'] ~ '/' ~ vapp['guestinfo.prefixlength']) | ansible.utils.ipaddr('netmask') }} \
+#         {{ vapp['guestinfo.dnsserver'] | split(',') | map('trim') | map('regex_replace', '^', '--dnsserver ') | join(' ') }} \
+#         --dnsdomain {{ vapp['metacluster.fqdn'] }} \
+#         --gateway {{ vapp['guestinfo.gateway'] }} \
+#         --force

ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/ippool.j2

@@ -0,0 +1,9 @@
+apiVersion: ipam.cluster.x-k8s.io/v1alpha1
+kind: InClusterIPPool
+metadata:
+  name: inclusterippool-{{ _template.cluster.name }}
+spec:
+  start: {{ _template.cluster.network.startip }}
+  end: {{ _template.cluster.network.endip }}
+  prefix: {{ _template.cluster.network.prefix }}
+  gateway: {{ _template.cluster.network.gateway }}

ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/kustomization.cluster-template.j2

@@ -35,12 +35,6 @@ patchesStrategicMerge:
       template:
         spec:
           files:
-          - encoding: base64
-            content: |
-              {{ _template.script.encoded }}
-            permissions: '0744'
-            owner: root:root
-            path: /root/network.sh
           - content: |
               network: {config: disabled}
             owner: root:root
@@ -49,6 +43,40 @@ patchesStrategicMerge:
               {{ _template.rootca | indent(width=14, first=False) | trim }}
             owner: root:root
             path: /usr/local/share/ca-certificates/root_ca.crt
+  - |-
+    apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+    kind: VSphereMachineTemplate
+    metadata:
+      name: ${CLUSTER_NAME}
+      namespace: '${NAMESPACE}'
+    spec:
+      template:
+        spec:
+          network:
+            devices:
+            - dhcp4: false
+              addressesFromPools:
+              - apiGroup: ipam.cluster.x-k8s.io
+                kind: InClusterIPPool
+                name: inclusterippool-${CLUSTER_NAME}
+              networkName: '${VSPHERE_NETWORK}'
+  - |-
+    apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+    kind: VSphereMachineTemplate
+    metadata:
+      name: ${CLUSTER_NAME}-worker
+      namespace: '${NAMESPACE}'
+    spec:
+      template:
+        spec:
+          network:
+            devices:
+            - dhcp4: false
+              addressesFromPools:
+              - apiGroup: ipam.cluster.x-k8s.io
+                kind: InClusterIPPool
+                name: inclusterippool-${CLUSTER_NAME}
+              networkName: '${VSPHERE_NETWORK}'
 
 patchesJson6902:
   - target:
@@ -57,15 +85,6 @@ patchesJson6902:
       kind: KubeadmControlPlane
       name: .*
     patch: |-
-      - op: add
-        path: /spec/kubeadmConfigSpec/files/-
-        value:
-          encoding: base64
-          content: |
-            {{ _template.script.encoded }}
-          owner: root:root
-          path: /root/network.sh
-          permissions: '0744'
       - op: add
         path: /spec/kubeadmConfigSpec/files/-
         value:

ansible/vars/metacluster.yml

@@ -296,8 +296,8 @@ dependencies:
       extra_opts: --strip-components=1
     - filename: kubectl-slice
       url: https://github.com/patrickdappollonio/kubectl-slice/releases/download/v1.2.5/kubectl-slice_linux_x86_64.tar.gz
-    - filename: npp-prepper
-      url: https://code.spamasaurus.com/api/packages/djpbessems/generic/npp-prepper/v0.5.1/npp-prepper
+    # - filename: npp-prepper
+    #   url: https://code.spamasaurus.com/api/packages/djpbessems/generic/npp-prepper/v0.5.1/npp-prepper
     - filename: skopeo
       url: https://code.spamasaurus.com/api/packages/djpbessems/generic/skopeo/v1.11.1/skopeo_linux_amd64
     - filename: step