From 92eee0744e21eb2eb26ea25acb029274d7d8b760 Mon Sep 17 00:00:00 2001 From: Danny Bessems Date: Mon, 28 Nov 2022 16:58:56 +0100 Subject: [PATCH] Attempt to simplify/aggregate dicts --- .../roles/assets/tasks/containerimages.yml | 98 ++++--------------- ansible/roles/assets/tasks/main.yml | 1 + ansible/roles/assets/tasks/manifests.yml | 65 ++++++++++++ 3 files changed, 84 insertions(+), 80 deletions(-) create mode 100644 ansible/roles/assets/tasks/manifests.yml diff --git a/ansible/roles/assets/tasks/containerimages.yml b/ansible/roles/assets/tasks/containerimages.yml index a50ff68..9828487 100644 --- a/ansible/roles/assets/tasks/containerimages.yml +++ b/ansible/roles/assets/tasks/containerimages.yml @@ -1,71 +1,10 @@ -- block: - - - name: Aggregate chart_values into dict - ansible.builtin.set_fact: - chart_values: "{{ chart_values | default({}) | combine({ (item.key | regex_replace('[^A-Za-z0-9]', '')): { 'chart_values': (item.value.helm.chart_values | from_yaml) } }) }}" - when: item.value.helm.chart_values is defined - loop: "{{ lookup('ansible.builtin.dict', components) }}" - loop_control: - label: "{{ item.key }}" - - - name: Write dict to vars_file - ansible.builtin.copy: - dest: /opt/firstboot/ansible/vars/metacluster.yml - content: "{{ { 'components': (chart_values | combine({ 'clusterapi': components.clusterapi })) } | to_nice_yaml(indent=2, width=4096) }}" - -- block: - - - name: Download ClusterAPI manifests - ansible.builtin.get_url: - url: "{{ item.url }}" - dest: /opt/metacluster/cluster-api/{{ item.dest }} - register: clusterapi_manifests - loop: - # This list is based on `clusterctl config repositories` - # Note: Each manifest also needs a `metadata.yaml` file stored in the respective folder - - url: https://github.com/kubernetes-sigs/cluster-api/releases/download/{{ components.clusterapi.management.version.base }}/bootstrap-components.yaml - dest: bootstrap-kubeadm/{{ components.clusterapi.management.version.base }}/bootstrap-components.yaml - - url: https://github.com/kubernetes-sigs/cluster-api/releases/download/{{ components.clusterapi.management.version.base }}/core-components.yaml - dest: cluster-api/{{ components.clusterapi.management.version.base }}/core-components.yaml - - url: https://github.com/kubernetes-sigs/cluster-api/releases/download/{{ components.clusterapi.management.version.base }}/control-plane-components.yaml - dest: control-plane-kubeadm/{{ components.clusterapi.management.version.base }}/control-plane-components.yaml - # This downloads the same `metadata.yaml` file to three separate folders - - url: https://github.com/kubernetes-sigs/cluster-api/releases/download/{{ components.clusterapi.management.version.base }}/metadata.yaml - dest: bootstrap-kubeadm/{{ components.clusterapi.management.version.base }}/metadata.yaml - - url: https://github.com/kubernetes-sigs/cluster-api/releases/download/{{ components.clusterapi.management.version.base }}/metadata.yaml - dest: cluster-api/{{ components.clusterapi.management.version.base }}/metadata.yaml - - url: https://github.com/kubernetes-sigs/cluster-api/releases/download/{{ components.clusterapi.management.version.base }}/metadata.yaml - dest: control-plane-kubeadm/{{ components.clusterapi.management.version.base }}/metadata.yaml - # The vsphere infrastructure provider requires multiple files (`cluster-template.yaml` and `metadata.yaml` on top of default files) - - url: https://github.com/kubernetes-sigs/cluster-api-provider-vsphere/releases/download/{{ components.clusterapi.management.version.infrastructure_vsphere }}/infrastructure-components.yaml - dest: infrastructure-vsphere/{{ components.clusterapi.management.version.infrastructure_vsphere }}/infrastructure-components.yaml - - url: https://github.com/kubernetes-sigs/cluster-api-provider-vsphere/releases/download/{{ components.clusterapi.management.version.infrastructure_vsphere }}/cluster-template.yaml - dest: infrastructure-vsphere/{{ components.clusterapi.management.version.infrastructure_vsphere }}/cluster-template.yaml - - url: https://github.com/kubernetes-sigs/cluster-api-provider-vsphere/releases/download/{{ components.clusterapi.management.version.infrastructure_vsphere }}/metadata.yaml - dest: infrastructure-vsphere/{{ components.clusterapi.management.version.infrastructure_vsphere }}/metadata.yaml - # Additionally, cert-manager is a prerequisite - - url: https://github.com/cert-manager/cert-manager/releases/download/{{ components.clusterapi.management.version.cert_manager }}/cert-manager.yaml - dest: cert-manager/{{ components.clusterapi.management.version.cert_manager }}/cert-manager.yaml - # Finally, workload clusters will need a CNI plugin - - url: https://raw.githubusercontent.com/projectcalico/calico/{{ components.clusterapi.workload.version.calico }}/manifests/calico.yaml - dest: cni-calico/{{ components.clusterapi.workload.version.calico }}/calico.yaml - loop_control: - label: "{{ item.url | basename }}" - retries: 5 - delay: 5 - until: clusterapi_manifests is not failed - - - name: Parse manifests - ansible.builtin.shell: - cmd: cat {{ item.dest }} | yq --no-doc eval '.. | .image? | select(.)' | awk '!/ /' | sort -u - register: parsedmanifests - loop: "{{ clusterapi_manifests.results }}" - loop_control: - label: "{{ item.dest | basename }}" - - - name: Store container images - ansible.builtin.set_fact: - clusterapi_containerimages: "{{ parsedmanifests | json_query('results[*].stdout_lines') | select() | flatten }}" +- name: Parse manifests for container images + ansible.builtin.shell: + cmd: cat {{ item.dest }} | yq --no-doc eval '.. | .image? | select(.)' | awk '!/ /' | sort -u + register: parsedmanifests + loop: "{{ clusterapi_manifests.results }}" + loop_control: + label: "{{ item.dest | basename }}" - name: Parse helm charts for container images ansible.builtin.shell: @@ -77,6 +16,16 @@ loop_control: label: "{{ item.key }}" +- name: Store container images in dicts + ansible.builtin.set_fact: + containerimages_{{ item.source }}: "{{ item.results }}" + loop: + - source: charts + # results: "{{ (containerimages.results | map(attribute='stdout_lines') | select('defined') | flatten }}" + results: "{{ containerimages | json_query('results[*].stdout_lines') | select() | flatten | list }}" + - source: manifests + results: "{{ parsedmanifests | json_query('results[*].stdout_lines') | select() | flatten | list }}" + - name: Pull and store containerimages ansible.builtin.shell: cmd: >- @@ -86,15 +35,4 @@ docker://{{ item }} \ docker-archive:./{{ ( item | regex_findall('[^/:]+'))[-2] }}_{{ lookup('ansible.builtin.password', '/dev/null length=5 chars=ascii_lowercase,digits seed={{ item }}') }}.tar:{{ item }} chdir: /opt/metacluster/container-images - loop: "{{ ((containerimages.results | map(attribute='stdout_lines') | select('defined') | flatten) + dependencies.container_images + (clusterapi_containerimages | list)) | unique }}" - -# - name: Inject manifests -# ansible.builtin.template: -# src: "{{ item.type }}.j2" -# dest: /var/lib/rancher/k3s/server/manifests/{{ item.name }}-manifest.yaml -# owner: root -# group: root -# mode: 0600 -# loop: "{{ lookup('ansible.builtin.dict', components) | map(attribute='value.manifests') | list | select('defined') | flatten }}" -# loop_control: -# label: "{{ item.type + '/' + item.name }}" + loop: "{{ containerimages_charts + containerimages_manifests + dependencies.container_images | unique }}" diff --git a/ansible/roles/assets/tasks/main.yml b/ansible/roles/assets/tasks/main.yml index 74681bf..f2f24fe 100644 --- a/ansible/roles/assets/tasks/main.yml +++ b/ansible/roles/assets/tasks/main.yml @@ -21,5 +21,6 @@ - import_tasks: k3s.yml - import_tasks: helm.yml - import_tasks: git.yml +- import_tasks: manifests.yml - import_tasks: containerimages.yml - import_tasks: nodetemplates.yml diff --git a/ansible/roles/assets/tasks/manifests.yml b/ansible/roles/assets/tasks/manifests.yml new file mode 100644 index 0000000..b809c78 --- /dev/null +++ b/ansible/roles/assets/tasks/manifests.yml @@ -0,0 +1,65 @@ +- block: + + - name: Aggregate chart_values into dict + ansible.builtin.set_fact: + chart_values: "{{ chart_values | default({}) | combine({ (item.key | regex_replace('[^A-Za-z0-9]', '')): { 'chart_values': (item.value.helm.chart_values | from_yaml) } }) }}" + when: item.value.helm.chart_values is defined + loop: "{{ lookup('ansible.builtin.dict', components) }}" + loop_control: + label: "{{ item.key }}" + + - name: Write dict to vars_file + ansible.builtin.copy: + dest: /opt/firstboot/ansible/vars/metacluster.yml + content: "{{ { 'components': (chart_values | combine({ 'clusterapi': components.clusterapi })) } | to_nice_yaml(indent=2, width=4096) }}" + +- name: Download ClusterAPI manifests + ansible.builtin.get_url: + url: "{{ item.url }}" + dest: /opt/metacluster/cluster-api/{{ item.dest }} + register: clusterapi_manifests + loop: + # This list is based on `clusterctl config repositories` + # Note: Each manifest also needs a `metadata.yaml` file stored in the respective folder + - url: https://github.com/kubernetes-sigs/cluster-api/releases/download/{{ components.clusterapi.management.version.base }}/bootstrap-components.yaml + dest: bootstrap-kubeadm/{{ components.clusterapi.management.version.base }}/bootstrap-components.yaml + - url: https://github.com/kubernetes-sigs/cluster-api/releases/download/{{ components.clusterapi.management.version.base }}/core-components.yaml + dest: cluster-api/{{ components.clusterapi.management.version.base }}/core-components.yaml + - url: https://github.com/kubernetes-sigs/cluster-api/releases/download/{{ components.clusterapi.management.version.base }}/control-plane-components.yaml + dest: control-plane-kubeadm/{{ components.clusterapi.management.version.base }}/control-plane-components.yaml + # This downloads the same `metadata.yaml` file to three separate folders + - url: https://github.com/kubernetes-sigs/cluster-api/releases/download/{{ components.clusterapi.management.version.base }}/metadata.yaml + dest: bootstrap-kubeadm/{{ components.clusterapi.management.version.base }}/metadata.yaml + - url: https://github.com/kubernetes-sigs/cluster-api/releases/download/{{ components.clusterapi.management.version.base }}/metadata.yaml + dest: cluster-api/{{ components.clusterapi.management.version.base }}/metadata.yaml + - url: https://github.com/kubernetes-sigs/cluster-api/releases/download/{{ components.clusterapi.management.version.base }}/metadata.yaml + dest: control-plane-kubeadm/{{ components.clusterapi.management.version.base }}/metadata.yaml + # The vsphere infrastructure provider requires multiple files (`cluster-template.yaml` and `metadata.yaml` on top of default files) + - url: https://github.com/kubernetes-sigs/cluster-api-provider-vsphere/releases/download/{{ components.clusterapi.management.version.infrastructure_vsphere }}/infrastructure-components.yaml + dest: infrastructure-vsphere/{{ components.clusterapi.management.version.infrastructure_vsphere }}/infrastructure-components.yaml + - url: https://github.com/kubernetes-sigs/cluster-api-provider-vsphere/releases/download/{{ components.clusterapi.management.version.infrastructure_vsphere }}/cluster-template.yaml + dest: infrastructure-vsphere/{{ components.clusterapi.management.version.infrastructure_vsphere }}/cluster-template.yaml + - url: https://github.com/kubernetes-sigs/cluster-api-provider-vsphere/releases/download/{{ components.clusterapi.management.version.infrastructure_vsphere }}/metadata.yaml + dest: infrastructure-vsphere/{{ components.clusterapi.management.version.infrastructure_vsphere }}/metadata.yaml + # Additionally, cert-manager is a prerequisite + - url: https://github.com/cert-manager/cert-manager/releases/download/{{ components.clusterapi.management.version.cert_manager }}/cert-manager.yaml + dest: cert-manager/{{ components.clusterapi.management.version.cert_manager }}/cert-manager.yaml + # Finally, workload clusters will need a CNI plugin + - url: https://raw.githubusercontent.com/projectcalico/calico/{{ components.clusterapi.workload.version.calico }}/manifests/calico.yaml + dest: cni-calico/{{ components.clusterapi.workload.version.calico }}/calico.yaml + loop_control: + label: "{{ item.url | basename }}" + retries: 5 + delay: 5 + until: clusterapi_manifests is not failed + +# - name: Inject manifests +# ansible.builtin.template: +# src: "{{ item.type }}.j2" +# dest: /var/lib/rancher/k3s/server/manifests/{{ item.name }}-manifest.yaml +# owner: root +# group: root +# mode: 0600 +# loop: "{{ lookup('ansible.builtin.dict', components) | map(attribute='value.manifests') | list | select('defined') | flatten }}" +# loop_control: +# label: "{{ item.type + '/' + item.name }}"