Simplify playbook roles

ansible/roles/assets/tasks/containerimages.yml

@@ -0,0 +1,97 @@
+- block:
+
+    - name: Aggregate chart_values into dict
+      ansible.builtin.set_fact:
+        chart_values: "{{ chart_values | default({}) | combine({ (item.key | regex_replace('[^A-Za-z0-9]', '')): { 'chart_values': (item.value.helm.chart_values | from_yaml) } }) }}"
+      when: item.value.helm.chart_values is defined
+      loop: "{{ lookup('ansible.builtin.dict', components) }}"
+      loop_control:
+        label: "{{ item.key }}"
+
+    - name: Write dict to vars_file
+      ansible.builtin.copy:
+        dest: /opt/firstboot/ansible/vars/metacluster.yml
+        content: "{{ { 'components': (chart_values | combine({ 'clusterapi': components.clusterapi })) } | to_nice_yaml(indent=2, width=4096) }}"
+
+- block:
+
+    - name: Download ClusterAPI manifests
+      ansible.builtin.get_url:
+        url: "{{ item.url }}"
+        dest: /opt/metacluster/cluster-api/{{ item.dest }}
+      register: clusterapi_manifests
+      loop:
+        # This list is based on `clusterctl config repositories`
+        # Note: Each manifest also needs a `metadata.yaml` file stored in the respective folder
+        - url: https://github.com/kubernetes-sigs/cluster-api/releases/download/{{ components.clusterapi.management.version.base }}/bootstrap-components.yaml
+          dest: bootstrap-kubeadm/{{ components.clusterapi.management.version.base }}/bootstrap-components.yaml
+        - url: https://github.com/kubernetes-sigs/cluster-api/releases/download/{{ components.clusterapi.management.version.base }}/core-components.yaml
+          dest: cluster-api/{{ components.clusterapi.management.version.base }}/core-components.yaml
+        - url: https://github.com/kubernetes-sigs/cluster-api/releases/download/{{ components.clusterapi.management.version.base }}/control-plane-components.yaml
+          dest: control-plane-kubeadm/{{ components.clusterapi.management.version.base }}/control-plane-components.yaml
+        # This downloads the same `metadata.yaml` file to three separate folders
+        - url: https://github.com/kubernetes-sigs/cluster-api/releases/download/{{ components.clusterapi.management.version.base }}/metadata.yaml
+          dest: bootstrap-kubeadm/{{ components.clusterapi.management.version.base }}/metadata.yaml
+        - url: https://github.com/kubernetes-sigs/cluster-api/releases/download/{{ components.clusterapi.management.version.base }}/metadata.yaml
+          dest: cluster-api/{{ components.clusterapi.management.version.base }}/metadata.yaml
+        - url: https://github.com/kubernetes-sigs/cluster-api/releases/download/{{ components.clusterapi.management.version.base }}/metadata.yaml
+          dest: control-plane-kubeadm/{{ components.clusterapi.management.version.base }}/metadata.yaml
+        # The vsphere infrastructure provider requires multiple files (`cluster-template.yaml` and `metadata.yaml` on top of default files)
+        - url: https://github.com/kubernetes-sigs/cluster-api-provider-vsphere/releases/download/{{ components.clusterapi.management.version.infrastructure_vsphere }}/infrastructure-components.yaml
+          dest: infrastructure-vsphere/{{ components.clusterapi.management.version.infrastructure_vsphere }}/infrastructure-components.yaml
+        - url: https://github.com/kubernetes-sigs/cluster-api-provider-vsphere/releases/download/{{ components.clusterapi.management.version.infrastructure_vsphere }}/cluster-template.yaml
+          dest: infrastructure-vsphere/{{ components.clusterapi.management.version.infrastructure_vsphere }}/cluster-template.yaml
+        - url: https://github.com/kubernetes-sigs/cluster-api-provider-vsphere/releases/download/{{ components.clusterapi.management.version.infrastructure_vsphere }}/metadata.yaml
+          dest: infrastructure-vsphere/{{ components.clusterapi.management.version.infrastructure_vsphere }}/metadata.yaml
+        # Additionally, cert-manager is a prerequisite
+        - url: https://github.com/cert-manager/cert-manager/releases/download/{{ components.clusterapi.management.version.cert_manager }}/cert-manager.yaml
+          dest: cert-manager/{{ components.clusterapi.management.version.cert_manager }}/cert-manager.yaml
+      loop_control:
+        label: "{{ item.url | basename }}"
+      retries: 5
+      delay: 5
+      until: clusterapi_manifests is not failed
+
+    - name: Parse manifests
+      ansible.builtin.shell:
+        cmd: cat {{ item.dest }} | yq --no-doc eval '.. | .image? | select(.)' | awk '!/ /' | sort
+      register: parsedmanifests
+      loop: "{{ clusterapi_manifests.results }}"
+      loop_control:
+        label: "{{ item.dest | basename }}"
+
+    - name: Store container images
+      ansible.builtin.set_fact:
+        clusterapi_containerimages: "{{ parsedmanifests | json_query('results[*].stdout_lines') | select() | flatten }}"
+
+- name: Parse helm charts for container images
+  ansible.builtin.shell:
+    cmd: "{{ item.value.helm.parse_logic }}"
+    chdir: /opt/metacluster/helm-charts/{{ item.key }}
+  register: containerimages
+  when: item.value.helm is defined
+  loop: "{{ lookup('ansible.builtin.dict', components) }}"
+  loop_control:
+    label: "{{ item.key }}"
+
+- name: Pull and store containerimages
+  ansible.builtin.shell:
+    cmd: >-
+      skopeo copy \
+        --insecure-policy \
+        --retry-times=5 \
+        docker://{{ item }} \
+        docker-archive:./{{ ( item | regex_findall('[^/:]+'))[-2] }}_{{ lookup('ansible.builtin.password', '/dev/null length=5 chars=ascii_lowercase,digits seed={{ item }}') }}.tar:{{ item }}
+    chdir: /opt/metacluster/container-images
+  loop: "{{ ((containerimages.results | map(attribute='stdout_lines') | select('defined') | flatten) + dependencies.container_images + (clusterapi_containerimages | list)) | unique }}"
+
+# - name: Inject manifests
+#   ansible.builtin.template:
+#     src: "{{ item.type }}.j2"
+#     dest: /var/lib/rancher/k3s/server/manifests/{{ item.name }}-manifest.yaml
+#     owner: root
+#     group: root
+#     mode: 0600
+#   loop: "{{ lookup('ansible.builtin.dict', components) | map(attribute='value.manifests') | list | select('defined') | flatten }}"
+#   loop_control:
+#     label: "{{ item.type + '/' + item.name }}"

ansible/roles/assets/tasks/dependencies.archive_compressed.yml

@@ -0,0 +1,27 @@
+---
+- name: Initialize tempfolder
+  ansible.builtin.tempfile:
+    state: directory
+  register: archive
+
+- name: Download & extract archived static binary
+  ansible.builtin.unarchive:
+    src: "{{ item.url }}"
+    dest: "{{ archive.path }}"
+    remote_src: yes
+    extra_opts: "{{ item.extra_opts | default(omit) }}"
+
+- name: Install extracted binary
+  ansible.builtin.copy:
+    src: "{{ archive.path }}/{{ item.filename }}"
+    dest: /usr/local/bin/{{ item.filename }}
+    remote_src: yes
+    owner: root
+    group: root
+    mode: 0755
+
+- name: Cleanup tempfolder
+  ansible.builtin.file:
+    path: "{{ archive.path }}"
+    state: absent
+  when: archive.path is defined

ansible/roles/assets/tasks/dependencies.yml

@@ -0,0 +1,50 @@
+- name: Download & install static binaries
+  ansible.builtin.get_url:
+    url: "{{ item.url }}"
+    url_username: "{{ item.username | default(omit) }}"
+    url_password: "{{ item.password | default(omit) }}"
+    dest: /usr/local/bin/{{ item.filename }}
+    owner: root
+    group: root
+    mode: 0755
+  loop: "{{ dependencies.static_binaries | selectattr('archive', 'undefined') }}"
+  loop_control:
+    label: "{{ item.filename }}"
+
+- name: Download, extract & install archived static binaries
+  include_tasks: dependencies.archive_compressed.yml
+  loop: "{{ dependencies.static_binaries | rejectattr('archive', 'undefined') | selectattr('archive', 'equalto', 'compressed') }}"
+  loop_control:
+    label: "{{ item.filename }}"
+
+- name: Install ansible-galaxy collections
+  ansible.builtin.shell:
+    cmd: ansible-galaxy collection install {{ item }}
+  register: collections
+  loop: "{{ dependencies.ansible_galaxy_collections }}"
+  retries: 5
+  delay: 5
+  until: collections is not failed
+
+- name: Install distro packages
+  ansible.builtin.apt:
+    pkg: "{{ dependencies.packages.apt }}"
+    state: latest
+    update_cache: yes
+    install_recommends: no
+
+- name: Upgrade all packages
+  ansible.builtin.apt:
+    name: '*'
+    state: latest
+    update_cache: yes
+
+- name: Install additional python packages
+  ansible.builtin.pip:
+    name: "{{ dependencies.packages.pip }}"
+    state: latest
+
+- name: Cleanup apt cache
+  ansible.builtin.apt:
+    autoremove: yes
+    purge: yes

ansible/roles/assets/tasks/git.yml

@@ -0,0 +1,5 @@
+- name: Clone git repository
+  ansible.builtin.git:
+    repo: "{{ platform.gitops.repository.uri }}"
+    version: "{{ platform.gitops.repository.revision }}"
+    dest: /opt/metacluster/git-repositories/gitops

ansible/roles/assets/tasks/helm.yml

@@ -0,0 +1,19 @@
+- name: Add helm repositories
+  kubernetes.core.helm_repository:
+    name: "{{ item.name }}"
+    repo_url: "{{ item.url }}"
+    state: present
+  loop: "{{ platform.helm_repositories }}"
+
+- name: Fetch helm charts
+  ansible.builtin.command:
+    cmd: helm fetch {{ item.value.helm.chart }} --untar --version {{ item.value.helm.version }}
+    chdir: /opt/metacluster/helm-charts
+  when: item.value.helm is defined
+  register: helmcharts
+  loop: "{{ lookup('ansible.builtin.dict', components) }}"
+  loop_control:
+    label: "{{ item.key }}"
+  retries: 5
+  delay: 5
+  until: helmcharts is not failed

ansible/roles/assets/tasks/k3s.yml

@@ -0,0 +1,44 @@
+
+- name: Download & install K3s binary
+  ansible.builtin.get_url:
+    url: https://github.com/k3s-io/k3s/releases/download/{{ platform.k3s.version }}/k3s
+    dest: /usr/local/bin/k3s
+    owner: root
+    group: root
+    mode: 0755
+  register: download
+  until: download is not failed
+  retries: 3
+  delay: 10
+
+- name: Download K3s images tarball
+  ansible.builtin.get_url:
+    url: https://github.com/k3s-io/k3s/releases/download/{{ platform.k3s.version }}/k3s-airgap-images-amd64.tar.gz
+    dest: /var/lib/rancher/k3s/agent/images
+  register: download
+  until: download is not failed
+  retries: 3
+  delay: 10
+
+- name: Download K3s install script
+  ansible.builtin.get_url:
+    url: https://get.k3s.io
+    dest: /opt/metacluster/k3s/install.sh
+    owner: root
+    group: root
+    mode: 0755
+  register: download
+  until: download is not failed
+  retries: 3
+  delay: 10
+
+- name: Inject manifests
+  ansible.builtin.template:
+    src: helmchartconfig.j2
+    dest: /var/lib/rancher/k3s/server/manifests/{{ item.name }}-config.yaml
+    owner: root
+    group: root
+    mode: 0600
+  loop: "{{ platform.packaged_components }}"
+  loop_control:
+    label: "{{ item.name }}"

ansible/roles/assets/tasks/main.yml

@@ -0,0 +1,24 @@
+- name: Create folder structure(s)
+  ansible.builtin.file:
+    path: "{{ item }}"
+    state: directory
+  loop:
+    - /opt/metacluster/cluster-api/bootstrap-kubeadm/{{ components.clusterapi.management.version.base }}
+    - /opt/metacluster/cluster-api/cert-manager/{{ components.clusterapi.management.version.cert_manager }}
+    - /opt/metacluster/cluster-api/cluster-api/{{ components.clusterapi.management.version.base }}
+    - /opt/metacluster/cluster-api/control-plane-kubeadm/{{ components.clusterapi.management.version.base }}
+    - /opt/metacluster/cluster-api/infrastructure-vsphere/{{ components.clusterapi.management.version.infrastructure_vsphere }}
+    - /opt/metacluster/container-images
+    - /opt/metacluster/git-repositories/gitops
+    - /opt/metacluster/helm-charts
+    - /opt/metacluster/k3s
+    - /opt/workloadcluster/node-templates
+    - /var/lib/rancher/k3s/agent/images
+    - /var/lib/rancher/k3s/server/manifests
+
+- import_tasks: dependencies.yml
+- import_tasks: k3s.yml
+- import_tasks: helm.yml
+- import_tasks: git
+- import_tasks: containerimages.yml
+- import_tasks: nodetemplates.yml

ansible/roles/assets/tasks/nodetemplates.yml

@@ -0,0 +1,4 @@
+- name: Download node-template image
+  ansible.builtin.uri:
+    url: "{{ components.clusterapi.workload.node_template.url }}"
+    dest: /opt/workloadcluster/node-templates/{{ components.clusterapi.workload.node_template.name }}