Attempt to simplify/aggregate dicts

ansible/roles/assets/tasks/containerimages.yml

@@ -1,71 +1,10 @@
-- block:
+- name: Parse manifests for container images
-
+  ansible.builtin.shell:
-    - name: Aggregate chart_values into dict
+    cmd: cat {{ item.dest }} | yq --no-doc eval '.. | .image? | select(.)' | awk '!/ /' | sort -u
-      ansible.builtin.set_fact:
+  register: parsedmanifests
-        chart_values: "{{ chart_values | default({}) | combine({ (item.key | regex_replace('[^A-Za-z0-9]', '')): { 'chart_values': (item.value.helm.chart_values | from_yaml) } }) }}"
+  loop: "{{ clusterapi_manifests.results }}"
-      when: item.value.helm.chart_values is defined
+  loop_control:
-      loop: "{{ lookup('ansible.builtin.dict', components) }}"
+    label: "{{ item.dest | basename }}"
-      loop_control:
-        label: "{{ item.key }}"
-
-    - name: Write dict to vars_file
-      ansible.builtin.copy:
-        dest: /opt/firstboot/ansible/vars/metacluster.yml
-        content: "{{ { 'components': (chart_values | combine({ 'clusterapi': components.clusterapi })) } | to_nice_yaml(indent=2, width=4096) }}"
-
-- block:
-
-    - name: Download ClusterAPI manifests
-      ansible.builtin.get_url:
-        url: "{{ item.url }}"
-        dest: /opt/metacluster/cluster-api/{{ item.dest }}
-      register: clusterapi_manifests
-      loop:
-        # This list is based on `clusterctl config repositories`
-        # Note: Each manifest also needs a `metadata.yaml` file stored in the respective folder
-        - url: https://github.com/kubernetes-sigs/cluster-api/releases/download/{{ components.clusterapi.management.version.base }}/bootstrap-components.yaml
-          dest: bootstrap-kubeadm/{{ components.clusterapi.management.version.base }}/bootstrap-components.yaml
-        - url: https://github.com/kubernetes-sigs/cluster-api/releases/download/{{ components.clusterapi.management.version.base }}/core-components.yaml
-          dest: cluster-api/{{ components.clusterapi.management.version.base }}/core-components.yaml
-        - url: https://github.com/kubernetes-sigs/cluster-api/releases/download/{{ components.clusterapi.management.version.base }}/control-plane-components.yaml
-          dest: control-plane-kubeadm/{{ components.clusterapi.management.version.base }}/control-plane-components.yaml
-        # This downloads the same `metadata.yaml` file to three separate folders
-        - url: https://github.com/kubernetes-sigs/cluster-api/releases/download/{{ components.clusterapi.management.version.base }}/metadata.yaml
-          dest: bootstrap-kubeadm/{{ components.clusterapi.management.version.base }}/metadata.yaml
-        - url: https://github.com/kubernetes-sigs/cluster-api/releases/download/{{ components.clusterapi.management.version.base }}/metadata.yaml
-          dest: cluster-api/{{ components.clusterapi.management.version.base }}/metadata.yaml
-        - url: https://github.com/kubernetes-sigs/cluster-api/releases/download/{{ components.clusterapi.management.version.base }}/metadata.yaml
-          dest: control-plane-kubeadm/{{ components.clusterapi.management.version.base }}/metadata.yaml
-        # The vsphere infrastructure provider requires multiple files (`cluster-template.yaml` and `metadata.yaml` on top of default files)
-        - url: https://github.com/kubernetes-sigs/cluster-api-provider-vsphere/releases/download/{{ components.clusterapi.management.version.infrastructure_vsphere }}/infrastructure-components.yaml
-          dest: infrastructure-vsphere/{{ components.clusterapi.management.version.infrastructure_vsphere }}/infrastructure-components.yaml
-        - url: https://github.com/kubernetes-sigs/cluster-api-provider-vsphere/releases/download/{{ components.clusterapi.management.version.infrastructure_vsphere }}/cluster-template.yaml
-          dest: infrastructure-vsphere/{{ components.clusterapi.management.version.infrastructure_vsphere }}/cluster-template.yaml
-        - url: https://github.com/kubernetes-sigs/cluster-api-provider-vsphere/releases/download/{{ components.clusterapi.management.version.infrastructure_vsphere }}/metadata.yaml
-          dest: infrastructure-vsphere/{{ components.clusterapi.management.version.infrastructure_vsphere }}/metadata.yaml
-        # Additionally, cert-manager is a prerequisite
-        - url: https://github.com/cert-manager/cert-manager/releases/download/{{ components.clusterapi.management.version.cert_manager }}/cert-manager.yaml
-          dest: cert-manager/{{ components.clusterapi.management.version.cert_manager }}/cert-manager.yaml
-        # Finally, workload clusters will need a CNI plugin
-        - url: https://raw.githubusercontent.com/projectcalico/calico/{{ components.clusterapi.workload.version.calico }}/manifests/calico.yaml
-          dest: cni-calico/{{ components.clusterapi.workload.version.calico }}/calico.yaml
-      loop_control:
-        label: "{{ item.url | basename }}"
-      retries: 5
-      delay: 5
-      until: clusterapi_manifests is not failed
-
-    - name: Parse manifests
-      ansible.builtin.shell:
-        cmd: cat {{ item.dest }} | yq --no-doc eval '.. | .image? | select(.)' | awk '!/ /' | sort -u
-      register: parsedmanifests
-      loop: "{{ clusterapi_manifests.results }}"
-      loop_control:
-        label: "{{ item.dest | basename }}"
-
-    - name: Store container images
-      ansible.builtin.set_fact:
-        clusterapi_containerimages: "{{ parsedmanifests | json_query('results[*].stdout_lines') | select() | flatten }}"
 
 - name: Parse helm charts for container images
   ansible.builtin.shell:
@@ -77,6 +16,16 @@
   loop_control:
     label: "{{ item.key }}"
 
+- name: Store container images in dicts
+  ansible.builtin.set_fact:
+    containerimages_{{ item.source }}: "{{ item.results }}"
+  loop:
+    - source: charts
+      # results: "{{ (containerimages.results | map(attribute='stdout_lines') | select('defined') | flatten }}"
+      results: "{{ containerimages | json_query('results[*].stdout_lines') | select() | flatten | list }}"
+    - source: manifests
+      results: "{{ parsedmanifests | json_query('results[*].stdout_lines') | select() | flatten | list }}"
+
 - name: Pull and store containerimages
   ansible.builtin.shell:
     cmd: >-
@@ -86,15 +35,4 @@
         docker://{{ item }} \
         docker-archive:./{{ ( item | regex_findall('[^/:]+'))[-2] }}_{{ lookup('ansible.builtin.password', '/dev/null length=5 chars=ascii_lowercase,digits seed={{ item }}') }}.tar:{{ item }}
     chdir: /opt/metacluster/container-images
-  loop: "{{ ((containerimages.results | map(attribute='stdout_lines') | select('defined') | flatten) + dependencies.container_images + (clusterapi_containerimages | list)) | unique }}"
+  loop: "{{ containerimages_charts + containerimages_manifests + dependencies.container_images | unique }}"
-
-# - name: Inject manifests
-#   ansible.builtin.template:
-#     src: "{{ item.type }}.j2"
-#     dest: /var/lib/rancher/k3s/server/manifests/{{ item.name }}-manifest.yaml
-#     owner: root
-#     group: root
-#     mode: 0600
-#   loop: "{{ lookup('ansible.builtin.dict', components) | map(attribute='value.manifests') | list | select('defined') | flatten }}"
-#   loop_control:
-#     label: "{{ item.type + '/' + item.name }}"

ansible/roles/assets/tasks/main.yml

@@ -21,5 +21,6 @@
 - import_tasks: k3s.yml
 - import_tasks: helm.yml
 - import_tasks: git.yml
+- import_tasks: manifests.yml
 - import_tasks: containerimages.yml
 - import_tasks: nodetemplates.yml

ansible/roles/assets/tasks/manifests.yml

@@ -0,0 +1,65 @@
+- block:
+
+    - name: Aggregate chart_values into dict
+      ansible.builtin.set_fact:
+        chart_values: "{{ chart_values | default({}) | combine({ (item.key | regex_replace('[^A-Za-z0-9]', '')): { 'chart_values': (item.value.helm.chart_values | from_yaml) } }) }}"
+      when: item.value.helm.chart_values is defined
+      loop: "{{ lookup('ansible.builtin.dict', components) }}"
+      loop_control:
+        label: "{{ item.key }}"
+
+    - name: Write dict to vars_file
+      ansible.builtin.copy:
+        dest: /opt/firstboot/ansible/vars/metacluster.yml
+        content: "{{ { 'components': (chart_values | combine({ 'clusterapi': components.clusterapi })) } | to_nice_yaml(indent=2, width=4096) }}"
+
+- name: Download ClusterAPI manifests
+  ansible.builtin.get_url:
+    url: "{{ item.url }}"
+    dest: /opt/metacluster/cluster-api/{{ item.dest }}
+  register: clusterapi_manifests
+  loop:
+    # This list is based on `clusterctl config repositories`
+    # Note: Each manifest also needs a `metadata.yaml` file stored in the respective folder
+    - url: https://github.com/kubernetes-sigs/cluster-api/releases/download/{{ components.clusterapi.management.version.base }}/bootstrap-components.yaml
+      dest: bootstrap-kubeadm/{{ components.clusterapi.management.version.base }}/bootstrap-components.yaml
+    - url: https://github.com/kubernetes-sigs/cluster-api/releases/download/{{ components.clusterapi.management.version.base }}/core-components.yaml
+      dest: cluster-api/{{ components.clusterapi.management.version.base }}/core-components.yaml
+    - url: https://github.com/kubernetes-sigs/cluster-api/releases/download/{{ components.clusterapi.management.version.base }}/control-plane-components.yaml
+      dest: control-plane-kubeadm/{{ components.clusterapi.management.version.base }}/control-plane-components.yaml
+    # This downloads the same `metadata.yaml` file to three separate folders
+    - url: https://github.com/kubernetes-sigs/cluster-api/releases/download/{{ components.clusterapi.management.version.base }}/metadata.yaml
+      dest: bootstrap-kubeadm/{{ components.clusterapi.management.version.base }}/metadata.yaml
+    - url: https://github.com/kubernetes-sigs/cluster-api/releases/download/{{ components.clusterapi.management.version.base }}/metadata.yaml
+      dest: cluster-api/{{ components.clusterapi.management.version.base }}/metadata.yaml
+    - url: https://github.com/kubernetes-sigs/cluster-api/releases/download/{{ components.clusterapi.management.version.base }}/metadata.yaml
+      dest: control-plane-kubeadm/{{ components.clusterapi.management.version.base }}/metadata.yaml
+    # The vsphere infrastructure provider requires multiple files (`cluster-template.yaml` and `metadata.yaml` on top of default files)
+    - url: https://github.com/kubernetes-sigs/cluster-api-provider-vsphere/releases/download/{{ components.clusterapi.management.version.infrastructure_vsphere }}/infrastructure-components.yaml
+      dest: infrastructure-vsphere/{{ components.clusterapi.management.version.infrastructure_vsphere }}/infrastructure-components.yaml
+    - url: https://github.com/kubernetes-sigs/cluster-api-provider-vsphere/releases/download/{{ components.clusterapi.management.version.infrastructure_vsphere }}/cluster-template.yaml
+      dest: infrastructure-vsphere/{{ components.clusterapi.management.version.infrastructure_vsphere }}/cluster-template.yaml
+    - url: https://github.com/kubernetes-sigs/cluster-api-provider-vsphere/releases/download/{{ components.clusterapi.management.version.infrastructure_vsphere }}/metadata.yaml
+      dest: infrastructure-vsphere/{{ components.clusterapi.management.version.infrastructure_vsphere }}/metadata.yaml
+    # Additionally, cert-manager is a prerequisite
+    - url: https://github.com/cert-manager/cert-manager/releases/download/{{ components.clusterapi.management.version.cert_manager }}/cert-manager.yaml
+      dest: cert-manager/{{ components.clusterapi.management.version.cert_manager }}/cert-manager.yaml
+    # Finally, workload clusters will need a CNI plugin
+    - url: https://raw.githubusercontent.com/projectcalico/calico/{{ components.clusterapi.workload.version.calico }}/manifests/calico.yaml
+      dest: cni-calico/{{ components.clusterapi.workload.version.calico }}/calico.yaml
+  loop_control:
+    label: "{{ item.url | basename }}"
+  retries: 5
+  delay: 5
+  until: clusterapi_manifests is not failed
+
+# - name: Inject manifests
+#   ansible.builtin.template:
+#     src: "{{ item.type }}.j2"
+#     dest: /var/lib/rancher/k3s/server/manifests/{{ item.name }}-manifest.yaml
+#     owner: root
+#     group: root
+#     mode: 0600
+#   loop: "{{ lookup('ansible.builtin.dict', components) | map(attribute='value.manifests') | list | select('defined') | flatten }}"
+#   loop_control:
+#     label: "{{ item.type + '/' + item.name }}"