Parallel build of bootstrap/upgrade ova;Split ansible tasks respectively

ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/filter_plugins/netaddr.py

@@ -0,0 +1,14 @@
+import netaddr
+
+def netaddr_iter_iprange(ip_start, ip_end):
+    return [str(ip) for ip in netaddr.iter_iprange(ip_start, ip_end)]
+
+class FilterModule(object):
+        ''' Ansible filter. Interface to netaddr methods.
+            https://pypi.org/project/netaddr/
+        '''
+
+        def filters(self):
+            return {
+                'netaddr_iter_iprange': netaddr_iter_iprange
+            }

ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/assets.yml

@@ -0,0 +1,12 @@
+- name: Import container images
+  ansible.builtin.command:
+    cmd: k3s ctr image import {{ item }} --digests
+    chdir: /opt/metacluster/container-images
+  register: import_result
+  loop: "{{ query('ansible.builtin.fileglob', '/opt/metacluster/container-images/*.tar') | sort }}"
+  loop_control:
+    label: "{{ item | basename }}"
+  # Probably should add a task before that ensures K3s node is fully initialized before starting imports; currently K3s goes away briefly during this loop
+  retries: "{{ playbook.retries }}"
+  delay: "{{ playbook.delays.short }}"
+  until: import_result is not failed

ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/certauthority.yml

@@ -0,0 +1,131 @@
+- block:
+
+    - name: Install step-ca chart
+      kubernetes.core.helm:
+        name: step-certificates
+        chart_ref: /opt/metacluster/helm-charts/step-certificates
+        release_namespace: step-ca
+        create_namespace: yes
+        # Unable to use REST api based readycheck due to missing ingress
+        wait: yes
+        kubeconfig: "{{ kubeconfig.path }}"
+        values: "{{ components.stepcertificates.chart_values }}"
+
+    - name: Retrieve configmap w/ root certificate
+      kubernetes.core.k8s_info:
+        kind: ConfigMap
+        name: step-certificates-certs
+        namespace: step-ca
+        kubeconfig: "{{ kubeconfig.path }}"
+      register: stepca_cm_certs
+
+    - name: Create target namespaces
+      kubernetes.core.k8s:
+        kind: Namespace
+        name: "{{ item }}"
+        state: present
+        kubeconfig: "{{ kubeconfig.path }}"
+      loop:
+        - argo-cd
+        # - kube-system
+
+    - name: Store root certificate in namespaced configmaps/secrets
+      kubernetes.core.k8s:
+        state: present
+        template: "{{ item.kind }}.j2"
+        kubeconfig: "{{ kubeconfig.path }}"
+      vars:
+        _template:
+          name: "{{ item.name }}"
+          namespace: "{{ item.namespace }}"
+          annotations: "{{ item.annotations | default('{}') | indent(width=4, first=True) }}"
+          labels: "{{ item.labels | default('{}') | indent(width=4, first=True) }}"
+          data: "{{ item.data }}"
+      loop:
+        - name: argocd-tls-certs-cm
+          namespace: argo-cd
+          kind: configmap
+          annotations: |
+            meta.helm.sh/release-name: argo-cd
+            meta.helm.sh/release-namespace: argo-cd
+          labels: |
+            app.kubernetes.io/managed-by: Helm
+            app.kubernetes.io/name: argocd-cm
+            app.kubernetes.io/part-of: argocd
+          data:
+            - key: git.{{ vapp['metacluster.fqdn'] }}
+              value: "{{ stepca_cm_certs.resources[0].data['root_ca.crt'] }}"
+        - name: step-certificates-certs
+          namespace: kube-system
+          kind: secret
+          data:
+            - key: root_ca.crt
+              value: "{{ stepca_cm_certs.resources[0].data['root_ca.crt'] | b64encode }}"
+      loop_control:
+        label: "{{ item.kind + '/' + item.name + ' (' + item.namespace + ')' }}"
+
+    - name: Configure step-ca passthrough ingress
+      ansible.builtin.template:
+        src: ingressroutetcp.j2
+        dest: /var/lib/rancher/k3s/server/manifests/{{ _template.name }}-manifest.yaml
+        owner: root
+        group: root
+        mode: 0600
+      vars:
+        _template:
+          name: step-ca
+          namespace: step-ca
+          config: |2
+              entryPoints:
+                - websecure
+              routes:
+              - match: HostSNI(`ca.{{ vapp['metacluster.fqdn'] }}`)
+                services:
+                - name: step-certificates
+                  port: 443
+              tls:
+                passthrough: true
+      notify:
+        - Apply manifests
+
+    - name: Inject step-ca certificate into traefik container
+      ansible.builtin.blockinfile:
+        path: /var/lib/rancher/k3s/server/manifests/traefik-config.yaml
+        block: |2
+              volumes:
+                - name: step-certificates-certs
+                  mountPath: /step-ca
+                  type: secret
+              env:
+                - name: LEGO_CA_CERTIFICATES
+                  value: /step-ca/root_ca.crt
+        marker: '    # {mark} ANSIBLE MANAGED BLOCK'
+      notify:
+        - Apply manifests
+
+    - name: Trigger handlers
+      ansible.builtin.meta: flush_handlers
+
+    - name: Retrieve step-ca configuration
+      kubernetes.core.k8s_info:
+        kind: ConfigMap
+        name: step-certificates-config
+        namespace: step-ca
+        kubeconfig: "{{ kubeconfig.path }}"
+      register: stepca_cm_config
+
+    - name: Install root CA in system truststore
+      ansible.builtin.shell:
+        cmd: >-
+          step ca bootstrap \
+            --ca-url=https://ca.{{ vapp['metacluster.fqdn'] }} \
+            --fingerprint={{ stepca_cm_config.resources[0].data['defaults.json'] | from_json | json_query('fingerprint') }} \
+            --install \
+            --force
+          update-ca-certificates
+
+  module_defaults:
+    ansible.builtin.uri:
+      validate_certs: no
+      status_code: [200, 201]
+      body_format: json

ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/git.yml

@@ -0,0 +1,139 @@
+- block:
+
+    - name: Install gitea chart
+      kubernetes.core.helm:
+        name: gitea
+        chart_ref: /opt/metacluster/helm-charts/gitea
+        release_namespace: gitea
+        create_namespace: yes
+        wait: no
+        kubeconfig: "{{ kubeconfig.path }}"
+        values: "{{ components.gitea.chart_values }}"
+
+    - name: Ensure gitea API availability
+      ansible.builtin.uri:
+        url: https://git.{{ vapp['metacluster.fqdn'] }}/api/healthz
+        method: GET
+      register: api_readycheck
+      until:
+        - api_readycheck.json.status is defined
+        - api_readycheck.json.status == 'pass'
+      retries: "{{ playbook.retries }}"
+      delay: "{{ playbook.delays.long }}"
+
+    - name: Configure additional SSH ingress
+      ansible.builtin.template:
+        src: ingressroutetcp.j2
+        dest: /var/lib/rancher/k3s/server/manifests/{{ _template.name }}-manifest.yaml
+        owner: root
+        group: root
+        mode: 0600
+      vars:
+        _template:
+          name: gitea-ssh
+          namespace: gitea
+          config: |2
+              entryPoints:
+                - ssh
+              routes:
+              - match: HostSNI(`*`)
+                services:
+                - name: gitea-ssh
+                  port: 22
+      notify:
+        - Apply manifests
+
+    - name: Trigger handlers
+      ansible.builtin.meta: flush_handlers
+
+    - name: Generate gitea API token
+      ansible.builtin.uri:
+        url: https://git.{{ vapp['metacluster.fqdn'] }}/api/v1/users/administrator/tokens
+        method: POST
+        user: administrator
+        password: "{{ vapp['metacluster.password'] }}"
+        force_basic_auth: yes
+        body:
+          name: token_init_{{ lookup('password', '/dev/null length=5 chars=ascii_letters,digits') }}
+      register: gitea_api_token
+
+    - name: Retrieve existing gitea configuration
+      ansible.builtin.uri:
+        url: https://git.{{ vapp['metacluster.fqdn'] }}/api/v1/repos/search
+        method: GET
+      register: gitea_existing_config
+
+    - block:
+
+        - name: Register SSH public key
+          ansible.builtin.uri:
+            url: https://git.{{ vapp['metacluster.fqdn'] }}/api/v1/user/keys
+            method: POST
+            headers:
+              Authorization: token {{ gitea_api_token.json.sha1 }}
+            body:
+              key: "{{ gitops_sshkey.public_key }}"
+              read_only: false
+              title: GitOps
+
+        - name: Create organization(s)
+          ansible.builtin.uri:
+            url: https://git.{{ vapp['metacluster.fqdn'] }}/api/v1/orgs
+            method: POST
+            headers:
+              Authorization: token {{ gitea_api_token.json.sha1 }}
+            body: "{{ item }}"
+          loop:
+            - full_name: Meta-cluster
+              description: Meta-cluster configuration items
+              username: mc
+              website: https://git.{{ vapp['metacluster.fqdn'] }}/mc
+              location: '[...]'
+              visibility: public
+            - full_name: Workload-cluster
+              description: Workload-cluster configuration items
+              username: wl
+              website: https://git.{{ vapp['metacluster.fqdn'] }}/wl
+              location: '[...]'
+              visibility: public
+          loop_control:
+            label: "{{ item.full_name }}"
+
+        - name: Create repositories
+          ansible.builtin.uri:
+            url: https://git.{{ vapp['metacluster.fqdn'] }}/api/v1/orgs/{{ item.organization }}/repos
+            method: POST
+            headers:
+              Authorization: token {{ gitea_api_token.json.sha1 }}
+            body: "{{ item.body }}"
+          loop:
+            - organization: mc
+              body:
+                name: GitOps.Config
+                # auto_init: true
+                # default_branch: main
+                description: GitOps manifests
+            - organization: wl
+              body:
+                name: Template.GitOps.Config
+                # auto_init: true
+                # default_branch: main
+                description: GitOps manifests
+          loop_control:
+            label: "{{ item.organization + '/' + item.body.name }}"
+
+        - name: Rebase/Push source gitops repository
+          ansible.builtin.shell:
+            cmd: |
+              git config --local http.sslVerify false
+              git remote set-url origin https://administrator:{{ vapp['metacluster.password'] | urlencode }}@git.{{ vapp['metacluster.fqdn'] }}/mc/GitOps.Config.git
+              git push
+            chdir: /opt/metacluster/git-repositories/gitops
+
+      when: (gitea_existing_config.json is undefined) or (gitea_existing_config.json.data | length == 0)
+
+  module_defaults:
+    ansible.builtin.uri:
+      validate_certs: no
+      status_code: [200, 201]
+      body_format: json

ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/gitops.yml

@@ -0,0 +1,70 @@
+- block:
+
+    - name: Install argo-cd chart
+      kubernetes.core.helm:
+        name: argo-cd
+        chart_ref: /opt/metacluster/helm-charts/argo-cd
+        release_namespace: argo-cd
+        create_namespace: yes
+        wait: no
+        kubeconfig: "{{ kubeconfig.path }}"
+        values: "{{ components.argocd.chart_values }}"
+
+    - name: Ensure argo-cd API availability
+      ansible.builtin.uri:
+        url: https://gitops.{{ vapp['metacluster.fqdn'] }}/api/version
+        method: GET
+      register: api_readycheck
+      until:
+        - api_readycheck.json.Version is defined
+      retries: "{{ playbook.retries }}"
+      delay: "{{ playbook.delays.long }}"
+
+    - name: Generate argo-cd API token
+      ansible.builtin.uri:
+        url: https://gitops.{{ vapp['metacluster.fqdn'] }}/api/v1/session
+        method: POST
+        force_basic_auth: yes
+        body:
+          username: admin
+          password: "{{ vapp['metacluster.password'] }}"
+      register: argocd_api_token
+
+    - name: Configure metacluster-gitops repository
+      ansible.builtin.template:
+        src: gitrepo.j2
+        dest: /var/lib/rancher/k3s/server/manifests/{{ _template.name }}-manifest.yaml
+        owner: root
+        group: root
+        mode: 0600
+      vars:
+        _template:
+          name: argocd-gitrepo-metacluster
+          namespace: argo-cd
+          uid: "{{ lookup('ansible.builtin.password', '/dev/null length=5 chars=ascii_lowercase,digits seed=inventory_hostname') }}"
+          privatekey: "{{ lookup('ansible.builtin.file', '~/.ssh/git_rsa_id') | indent(4, true) }}"
+      notify:
+        - Apply manifests
+
+    - name: Create applicationset
+      ansible.builtin.template:
+        src: applicationset.j2
+        dest: /var/lib/rancher/k3s/server/manifests/{{ _template.name }}-manifest.yaml
+        owner: root
+        group: root
+        mode: 0600
+      vars:
+        _template:
+          name: argocd-applicationset-metacluster
+          namespace: argo-cd
+      notify:
+        - Apply manifests
+
+    - name: Trigger handlers
+      ansible.builtin.meta: flush_handlers
+
+  module_defaults:
+    ansible.builtin.uri:
+      validate_certs: no
+      status_code: [200, 201]
+      body_format: json

ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/ingress.yml

@@ -0,0 +1,26 @@
+- name: Configure traefik dashboard ingress
+  ansible.builtin.template:
+    src: ingressroute.j2
+    dest: /var/lib/rancher/k3s/server/manifests/{{ _template.name }}-manifest.yaml
+    owner: root
+    group: root
+    mode: 0600
+  vars:
+    _template:
+      name: traefik-dashboard
+      namespace: kube-system
+      config: |2
+          entryPoints:
+          - web
+          - websecure
+          routes:
+          - kind: Rule
+            match: Host(`ingress.{{ vapp['metacluster.fqdn'] }}`)
+            services:
+            - kind: TraefikService
+              name: api@internal
+  notify:
+    - Apply manifests
+
+- name: Trigger handlers
+  ansible.builtin.meta: flush_handlers

ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/init.yml

@@ -0,0 +1,13 @@
+- name: Configure fallback name resolution
+  ansible.builtin.lineinfile:
+    path: /etc/hosts
+    line: "{{ vapp['guestinfo.ipaddress'] }}  {{ item + '.' + vapp['metacluster.fqdn'] }}"
+    state: present
+  loop:
+    # TODO: Make this list dynamic
+    - ca
+    - git
+    - gitops
+    - ingress
+    - registry
+    - storage

ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/k3s.yml

@@ -0,0 +1,74 @@
+- name: Store custom configuration files
+  ansible.builtin.copy:
+    dest: "{{ item.filename }}"
+    content: "{{ item.content }}"
+  loop:
+    - filename: /etc/rancher/k3s/config.yaml
+      content: |
+        kubelet-arg:
+          - "config=/etc/rancher/k3s/kubelet.config"
+    - filename: /etc/rancher/k3s/kubelet.config
+      content: |
+        apiVersion: kubelet.config.k8s.io/v1beta1
+        kind: KubeletConfiguration
+
+        shutdownGracePeriod: 180s
+        shtudownGracePeriodCriticalPods: 60s
+  loop_control:
+    label: "{{ item.filename }}"
+
+- name: Gather service facts
+  ansible.builtin.service_facts:
+    # Module requires no attributes
+
+- name: Install K3s
+  ansible.builtin.command:
+    cmd: ./install.sh
+    chdir: /opt/metacluster/k3s
+  environment:
+    INSTALL_K3S_SKIP_DOWNLOAD: 'true'
+    INSTALL_K3S_EXEC: 'server --cluster-init --disable local-storage --config /etc/rancher/k3s/config.yaml'
+  when: ansible_facts.services['k3s.service'] is undefined
+
+- name: Debug possible taints on k3s node
+  ansible.builtin.shell:
+    cmd: >-
+      while true;
+      do
+        kubectl get nodes -o custom-columns=NAME:.metadata.name,TAINTS:.spec.taints --no-headers | awk '{print strftime("%H:%M:%S"),$0;fflush();}' >> /var/log/taintlog
+        sleep 1
+      done
+  async: 1800
+  poll: 0
+
+- name: Ensure API availability
+  ansible.builtin.uri:
+    url: https://{{ vapp['guestinfo.ipaddress'] }}:6443/livez?verbose
+    method: GET
+    validate_certs: no
+    status_code: [200, 401]
+  register: api_readycheck
+  until: api_readycheck.json.apiVersion is defined
+  retries: "{{ playbook.retries }}"
+  delay: "{{ playbook.delays.medium }}"
+
+- name: Install kubectl tab-completion
+  ansible.builtin.shell:
+    cmd: kubectl completion bash | tee /etc/bash_completion.d/kubectl
+
+- name: Initialize tempfile
+  ansible.builtin.tempfile:
+    state: file
+  register: kubeconfig
+
+- name: Retrieve kubeconfig
+  ansible.builtin.command:
+    cmd: kubectl config view --raw
+  register: kubectl_config
+
+- name: Store kubeconfig in tempfile
+  ansible.builtin.copy:
+    dest: "{{ kubeconfig.path }}"
+    content: "{{ kubectl_config.stdout }}"
+    mode: 0600
+  no_log: true

ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/main.yml

@@ -0,0 +1,9 @@
+- import_tasks: init.yml
+- import_tasks: k3s.yml
+- import_tasks: assets.yml
+- import_tasks: ingress.yml
+- import_tasks: storage.yml
+- import_tasks: certauthority.yml
+- import_tasks: registry.yml
+- import_tasks: git.yml
+- import_tasks: gitops.yml

ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/registry.yml

@@ -0,0 +1,85 @@
+- block:
+
+    - name: Install harbor chart
+      kubernetes.core.helm:
+        name: harbor
+        chart_ref: /opt/metacluster/helm-charts/harbor
+        release_namespace: harbor
+        create_namespace: yes
+        wait: no
+        kubeconfig: "{{ kubeconfig.path }}"
+        values: "{{ components.harbor.chart_values }}"
+
+    - name: Ensure harbor API availability
+      ansible.builtin.uri:
+        url: https://registry.{{ vapp['metacluster.fqdn'] }}/api/v2.0/health
+        method: GET
+      register: api_readycheck
+      until:
+        - api_readycheck.json.status is defined
+        - api_readycheck.json.status == 'healthy'
+      retries: "{{ playbook.retries }}"
+      delay: "{{ playbook.delays.long }}"
+
+    - name: Push images to registry
+      ansible.builtin.shell:
+        cmd: >-
+          skopeo copy \
+            --insecure-policy \
+            --dest-tls-verify=false \
+            --dest-creds admin:{{ vapp['metacluster.password'] }} \
+            docker-archive:./{{ item | basename }} \
+            docker://registry.{{ vapp['metacluster.fqdn'] }}/library/$( \
+              skopeo list-tags \
+                --insecure-policy \
+                docker-archive:./{{ item | basename }} | \
+              jq -r '.Tags[0]')
+        chdir: /opt/metacluster/container-images/
+      register: push_result
+      loop: "{{ query('ansible.builtin.fileglob', '/opt/metacluster/container-images/*.tar') | sort }}"
+      loop_control:
+        label: "{{ item | basename }}"
+      retries: "{{ playbook.retries }}"
+      delay: "{{ playbook.delays.short }}"
+      until: push_result is not failed
+
+    - name: Get all stored container images (=artifacts)
+      ansible.builtin.uri:
+        url: https://registry.{{ vapp['metacluster.fqdn'] }}/api/v2.0/search?q=library
+        method: GET
+      register: registry_artifacts
+
+    - name: Get source registries of all artifacts
+      ansible.builtin.set_fact:
+        source_registries: "{{ (source_registries | default([]) + [(item | split('/'))[1]]) | unique | sort }}"
+      loop: "{{ registry_artifacts.json.repository | json_query('[*].repository_name') }}"
+
+    - name: Configure K3s node for private registry
+      ansible.builtin.template:
+        dest: /etc/rancher/k3s/registries.yaml
+        src: registries.j2
+      vars:
+        _template:
+          data: "{{ source_registries }}"
+          hv:
+            fqdn: "{{ vapp['metacluster.fqdn'] }}"
+
+    # - name: Restart kubelet (k3s) to pick up configured registries
+    #   ansible.builtin.systemd:
+    #     name: k3s
+    #     state: restarted
+
+    # - name: Ensure k3s API availability
+    #   ansible.builtin.uri:
+    #     url: https://{{ vapp['guestinfo.ipaddress'] }}:6443/livez?verbose
+    #     method: GET
+    #   register: api_readycheck
+    #   until: api_readycheck.json.apiVersion is defined
+    #   retries: 5
+    #   delay: 30
+
+  module_defaults:
+    ansible.builtin.uri:
+      validate_certs: no
+      status_code: [200, 201, 401]
+      body_format: json

ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/storage.yml

@@ -0,0 +1,26 @@
+- block:
+    - name: Install longhorn chart
+      kubernetes.core.helm:
+        name: longhorn
+        chart_ref: /opt/metacluster/helm-charts/longhorn
+        release_namespace: longhorn-system
+        create_namespace: yes
+        wait: no
+        kubeconfig: "{{ kubeconfig.path }}"
+        values: "{{ components.longhorn.chart_values }}"
+
+    - name: Ensure longhorn API availability
+      ansible.builtin.uri:
+        url: https://storage.{{ vapp['metacluster.fqdn'] }}/v1
+        method: GET
+      register: api_readycheck
+      until:
+        - api_readycheck is not failed
+      retries: "{{ playbook.retries }}"
+      delay: "{{ playbook.delays.long }}"
+
+  module_defaults:
+    ansible.builtin.uri:
+      validate_certs: no
+      status_code: [200, 201]
+      body_format: json