Disable Harbor tls (rely on Traefik);Configure Traefik with custom certResolver;Retrieve & install root ca in truststore

.drone.yml

@@ -14,29 +14,32 @@ steps:
 - name: Debugging information
   image: bv11-cr01.bessems.eu/library/packer-extended
   commands:
-  - yamllint --version
-  - packer --version
   - ansible --version
   - ovftool --version
-- name: Ubuntu Server 20.04
+  - packer --version
+  - yamllint --version
+- name: Kubernetes Bootstrap Appliance
   image: bv11-cr01.bessems.eu/library/packer-extended
   pull: always
   commands:
   - |
     sed -i -e "s/<<img-password>>/$${SSH_PASSWORD}/g" \
-      packer/preseed/UbuntuServer20.04/user-data
+      packer/preseed/UbuntuServer22.04/user-data
   - |
     yamllint -d "{extends: relaxed, rules: {line-length: disable}}" \
       ansible \
-      packer/preseed/UbuntuServer20.04/user-data \
+      packer/preseed/UbuntuServer22.04/user-data \
       scripts
+  - |
+    ansible-galaxy install \
+      -r ansible/requirements.yml
   - |
     packer init -upgrade \
       ./packer
   - |
     packer validate \
       -var vm_name=$DRONE_BUILD_NUMBER-${DRONE_COMMIT_SHA:0:10} \
-      -var vm_guestos=ubuntuserver20.04 \
+      -var vm_guestos=k8sbootstrap \
       -var repo_username=$${REPO_USERNAME} \
       -var repo_password=$${REPO_PASSWORD} \
       -var vsphere_password=$${VSPHERE_PASSWORD} \
@@ -46,7 +49,7 @@ steps:
     packer build \
       -on-error=cleanup -timestamp-ui \
       -var vm_name=$DRONE_BUILD_NUMBER-${DRONE_COMMIT_SHA:0:10} \
-      -var vm_guestos=ubuntuserver20.04 \
+      -var vm_guestos=k8sbootstrap \
       -var repo_username=$${REPO_USERNAME} \
       -var repo_password=$${REPO_PASSWORD} \
       -var vsphere_password=$${VSPHERE_PASSWORD} \

README.md

@@ -1 +1 @@
-# Packer.Images [![Build Status](https://ci.spamasaurus.com/api/badges/djpbessems/Packer.Images/status.svg?ref=refs/heads/Windows10)](https://ci.spamasaurus.com/djpbessems/Packer.Images)
+# Packer.Images [![Build Status](https://ci.spamasaurus.com/api/badges/djpbessems/Packer.Images/status.svg?ref=refs/heads/Kubernetes.Bootstrap.Appliance)](https://ci.spamasaurus.com/djpbessems/Packer.Images)

ansible/playbook.yml

@@ -1,7 +1,12 @@
 ---
 - hosts: all
   gather_facts: false
+  vars_files:
+    - metacluster.yml
+    - workloadcluster.yml
   become: true
   roles:
     - os
     - firstboot
+    - appliance
+    - metacluster

ansible/requirements.yml

@@ -0,0 +1,4 @@
+collections:
+  - ansible.utils
+  - community.general
+  - kubernetes.core

ansible/roles/appliance/tasks/dependencies.archive_compressed.yml

@@ -0,0 +1,27 @@
+---
+- name: Initialize tempfolder
+  ansible.builtin.tempfile:
+    state: directory
+  register: archive
+
+- name: Download & extract archived static binary
+  ansible.builtin.unarchive:
+    src: "{{ item.url }}"
+    dest: "{{ archive.path }}"
+    remote_src: yes
+    extra_opts: "{{ item.extra_opts | default(omit) }}"
+
+- name: Install extracted binary
+  ansible.builtin.copy:
+    src: "{{ archive.path }}/{{ item.filename }}"
+    dest: /usr/local/bin/{{ item.filename }}
+    remote_src: yes
+    owner: root
+    group: root
+    mode: 0755
+
+- name: Cleanup tempfolder
+  ansible.builtin.file:
+    path: "{{ archive.path }}"
+    state: absent
+  when: archive.path is defined

ansible/roles/appliance/tasks/dependencies.yml

@@ -0,0 +1,53 @@
+# - name: Create folder structure(s)
+#  ansible.builtin.file:
+#    path: "{{ item }}"
+#    state: directory
+#  loop:
+#    - /foo
+
+- name: Download & install static binaries
+  ansible.builtin.get_url:
+    url: "{{ item.url }}"
+    url_username: "{{ item.username | default(omit) }}"
+    url_password: "{{ item.password | default(omit) }}"
+    dest: /usr/local/bin/{{ item.filename }}
+    owner: root
+    group: root
+    mode: 0755
+  loop: "{{ dependencies.static_binaries | selectattr('archive', 'undefined') }}"
+  loop_control:
+    label: "{{ item.filename }}"
+
+- name: Download, extract & install archived static binaries
+  include_tasks: dependencies.archive_compressed.yml
+  loop: "{{ dependencies.static_binaries | rejectattr('archive', 'undefined') | selectattr('archive', 'equalto', 'compressed') }}"
+  loop_control:
+    label: "{{ item.filename }}"
+
+- name: Install ansible-galaxy collections
+  ansible.builtin.shell:
+    cmd: ansible-galaxy collection install {{ item }}
+  loop: "{{ dependencies.ansible_galaxy_collections }}"
+
+- name: Install distro packages
+  ansible.builtin.apt:
+    pkg: "{{ dependencies.packages.apt }}"
+    state: latest
+    update_cache: yes
+    install_recommends: no
+
+- name: Upgrade all packages
+  ansible.builtin.apt:
+    name: '*'
+    state: latest
+    update_cache: yes
+
+- name: Install additional python packages
+  ansible.builtin.pip:
+    name: "{{ dependencies.packages.pip }}"
+    state: latest
+
+- name: Cleanup apt cache
+  ansible.builtin.apt:
+    autoremove: yes
+    purge: yes

ansible/roles/appliance/tasks/main.yml

@@ -0,0 +1,2 @@
+- name: Install & configure dependencies
+  import_tasks: dependencies.yml

ansible/roles/firstboot/files/ansible_payload/playbook.yml

@@ -2,9 +2,23 @@
 - hosts: 127.0.0.1
   connection: local
   gather_facts: false
+  vars_files:
+    - metacluster.yml
   # become: true
   roles:
     - vapp
     - network
     - users
+    - disks
+    - metacluster
+    - workloadcluster
+    - tty
     - cleanup
+  handlers:
+    - name: Apply manifests
+      kubernetes.core.k8s:
+        src: "{{ item }}"
+        state: present
+        kubeconfig: "{{ kubeconfig.path }}"
+      with_fileglob: /var/lib/rancher/k3s/server/manifests/*.yaml
+      ignore_errors: yes

ansible/roles/firstboot/files/ansible_payload/roles/cleanup/tasks/main.yml

@@ -2,19 +2,7 @@
   ansible.builtin.cron:
     name: firstboot
     state: absent
-- name: Restore extra tty
-  ansible.builtin.lineinfile:
-    path: /etc/systemd/logind.conf
-    regexp: "{{ item.regexp }}"
-    line: "{{ item.line }}"
-  loop:
-  - { regexp: '^NAutoVTs=', line: '#NAutoVTs=6'}
-  - { regexp: '^ReserveVT=', line: '#ReserveVT=6'}
-- name: Unmask getty@tty1 service
-  ansible.builtin.systemd:
-    name: getty@tty1
-    enabled: yes
-    masked: no
+
 - name: Reboot host
   ansible.builtin.shell:
     cmd: /usr/sbin/reboot now

ansible/roles/firstboot/files/ansible_payload/roles/disks/tasks/main.yml

@@ -0,0 +1,24 @@
+- name: Create volume group
+  community.general.lvg:
+    vg: longhorn_vg
+    pvs:
+      - /dev/sdb
+    pvresize: yes
+
+- name: Create logical volume
+  community.general.lvol:
+    vg: longhorn_vg
+    lv: longhorn_lv
+    size: 100%VG
+
+- name: Create filesystem
+  community.general.filesystem:
+    dev: /dev/mapper/longhorn_vg-longhorn_lv
+    fstype: ext4
+
+- name: Mount dynamic disk
+  ansible.posix.mount:
+    path: /mnt/blockstorage
+    src: /dev/mapper/longhorn_vg-longhorn_lv
+    fstype: ext4
+    state: mounted

ansible/roles/firstboot/files/ansible_payload/roles/metacluster/tasks/main.yml

@@ -0,0 +1,398 @@
+- name: Gather service facts
+  ansible.builtin.service_facts:
+    # Module requires no attributes
+
+- name: Install K3s
+  ansible.builtin.command:
+    cmd: ./install.sh
+    chdir: /opt/metacluster/k3s
+  environment:
+    INSTALL_K3S_SKIP_DOWNLOAD: 'true'
+    INSTALL_K3S_EXEC: 'server --cluster-init --disable local-storage'
+  when: ansible_facts.services['k3s.service'] is undefined
+
+- name: Ensure API availability
+  ansible.utils.cli_parse:
+    command: curl -k https://{{ vapp['guestinfo.ipaddress'] }}:6443/livez?verbose
+    parser:
+      name: ansible.utils.json
+    set_fact: api_readycheck
+  ignore_errors: yes
+  until: api_readycheck.apiVersion is defined
+  retries: 3
+  delay: 30
+
+- name: Install kubectl tab-completion
+  ansible.builtin.shell:
+    cmd: kubectl completion bash | tee /etc/bash_completion.d/kubectl
+
+- name: Extract container images
+  ansible.builtin.unarchive:
+    src: /opt/metacluster/container-images/image-tarballs.tgz
+    dest: /opt/metacluster/container-images
+    list_files: yes
+  register: imagetarballs
+
+- name: Import container images
+  ansible.builtin.command:
+    cmd: k3s ctr image import {{ item }}
+    chdir: /opt/metacluster/container-images
+  loop: "{{ imagetarballs.files }}"
+
+- name: Initialize tempfile
+  ansible.builtin.tempfile:
+    state: file
+  register: kubeconfig
+
+- name: Retrieve kubeconfig
+  ansible.builtin.command:
+    cmd: kubectl config view --raw
+  register: kubectl_config
+
+- name: Store kubeconfig in tempfile
+  ansible.builtin.copy:
+    dest: "{{ kubeconfig.path }}"
+    content: "{{ kubectl_config.stdout }}"
+    mode: 0600
+  no_log: true
+
+- name: Configure traefik dashboard ingress
+  ansible.builtin.template:
+    src: ingressroute.j2
+    dest: /var/lib/rancher/k3s/server/manifests/{{ _template.name }}-manifest.yaml
+    owner: root
+    group: root
+    mode: 0600
+  vars:
+    _template:
+      name: traefik-dashboard
+      namespace: kube-system
+      config: |2
+          entryPoints:
+          - web
+          - websecure
+          routes:
+          - kind: Rule
+            match: Host(`ingress.{{ vapp['metacluster.fqdn'] }}`)
+            services:
+            - kind: TraefikService
+              name: api@internal
+  notify:
+    - Apply manifests
+
+- name: Trigger handlers
+  ansible.builtin.meta: flush_handlers
+
+- name: Create component entries in /etc/hosts
+  ansible.builtin.lineinfile:
+    path: /etc/hosts
+    line: "{{ vapp['guestinfo.ipaddress'] }}  {{ item + '.' + vapp['metacluster.fqdn'] }}"
+    state: present
+  loop:
+    # TODO: Make this list dynamic
+    - git
+    - gitops
+    - ingress
+    - registry
+    - storage
+
+- name: Install longhorn chart
+  kubernetes.core.helm:
+    name: longhorn
+    chart_ref: /opt/metacluster/helm-charts/longhorn
+    release_namespace: longhorn-system
+    create_namespace: yes
+    wait: yes
+    kubeconfig: "{{ kubeconfig.path }}"
+    values: "{{ components.longhorn.chart_values }}"
+
+- name: Install step-ca chart
+  kubernetes.core.helm:
+    name: step-certificates
+    chart_ref: /opt/metacluster/helm-charts/step-certificates
+    release_namespace: step-ca
+    create_namespace: yes
+    wait: yes
+    kubeconfig: "{{ kubeconfig.path }}"
+    values: "{{ components.stepcertificates.chart_values }}"
+
+- name: Retrieve step-ca configuration
+  kubernetes.core.k8s_log:
+    kind: Job
+    name: step-certificates
+    namespace: step-ca
+    kubeconfig: "{{ kubeconfig.path }}"
+  register: stepca_bootstraplog
+
+- name: Install root CA in system truststore
+  ansible.builtin.shell:
+    cmd: |
+      step ca bootstrap \
+        --ca-url={{ stepca_bootstraplog.log | regex_search('CA URL: (.+)', '\\1') | first }} \
+        --fingerprint={{ stepca_bootstraplog.log | regex_search('CA Fingerprint: (.+)', '\\1') | first }} \
+        --install
+
+- name: Install harbor chart
+  kubernetes.core.helm:
+    name: harbor
+    chart_ref: /opt/metacluster/helm-charts/harbor
+    release_namespace: harbor
+    create_namespace: yes
+    wait: yes
+    kubeconfig: "{{ kubeconfig.path }}"
+    values: "{{ components.harbor.chart_values }}"
+
+- name: Push images to registry
+  ansible.builtin.shell:
+    cmd: >-
+      skopeo copy \
+        --insecure-policy \
+        --dest-tls-verify=false \
+        --dest-creds admin:{{ vapp['guestinfo.rootpw'] }} \
+        docker-archive:./{{ item }} \
+        docker://registry.{{ vapp['metacluster.fqdn'] }}/library/$( \
+          skopeo list-tags \
+            --insecure-policy \
+            docker-archive:./{{ item }} | \
+          jq -r '.Tags[0]')
+    chdir: /opt/metacluster/container-images/
+  loop: "{{ imagetarballs.files }}"
+
+- name: Configure K3s node for private registry
+  ansible.builtin.template:
+    dest: /etc/rancher/k3s/registries.yaml
+    src: registries.j2
+  notify:
+    - Apply manifests
+
+- name: Trigger handlers
+  ansible.builtin.meta: flush_handlers
+
+- block:
+
+    - name: Install gitea chart
+      kubernetes.core.helm:
+        name: gitea
+        chart_ref: /opt/metacluster/helm-charts/gitea
+        release_namespace: gitea
+        create_namespace: yes
+        wait: yes
+        kubeconfig: "{{ kubeconfig.path }}"
+        values: "{{ components.gitea.chart_values }}"
+
+    - name: Configure additional SSH ingress
+      ansible.builtin.template:
+        src: ingressroutetcp.j2
+        dest: /var/lib/rancher/k3s/server/manifests/{{ _template.name }}-manifest.yaml
+        owner: root
+        group: root
+        mode: 0600
+      vars:
+        _template:
+          name: gitea-ssh
+          namespace: gitea
+          config: |2
+              entryPoints:
+                - ssh
+              routes:
+              - match: HostSNI(`*`)
+                services:
+                - name: gitea-ssh
+                  port: 22
+      notify:
+        - Apply manifests
+
+    - name: Trigger handlers
+      ansible.builtin.meta: flush_handlers
+
+    - name: Ensure gitea API availability
+      ansible.utils.cli_parse:
+        # Available from Gitea 1.17.x
+        # command: curl -k https://git.{{ vapp['metacluster.fqdn'] }}/api/healtz
+        command: curl -k https://git.{{ vapp['metacluster.fqdn'] }}/api/v1/version
+        parser:
+          name: ansible.utils.json
+        set_fact: api_readycheck
+      ignore_errors:
+      until: api_readycheck.version is defined
+      retries: 3
+      delay: 30
+
+    - name: Generate gitea API token
+      ansible.builtin.uri:
+        url: https://git.{{ vapp['metacluster.fqdn'] }}/api/v1/users/administrator/tokens
+        method: POST
+        user: administrator
+        password: "{{ vapp['guestinfo.rootpw'] }}"
+        force_basic_auth: yes
+        body:
+          name: token_init_{{ lookup('password', '/dev/null length=5 chars=ascii_letters,digits') }}
+      register: gitea_api_token
+
+    - name: Retrieve existing gitea configuration
+      ansible.builtin.uri:
+        url: https://git.{{ vapp['metacluster.fqdn'] }}/api/v1/repos/search
+        method: GET
+      register: gitea_existing_config
+
+    - ansible.builtin.uri:
+        url: "{{ item }}"
+        method: GET
+        headers:
+          Authorization: token {{ gitea_api_token.json.sha1 }}
+      register: gitea_existing_config_TST
+      loop:
+        # - https://git.{{ vapp['metacluster.fqdn'] }}/api/v1/orgs
+        - https://git.{{ vapp['metacluster.fqdn'] }}/api/v1/repos/search
+        - https://git.{{ vapp['metacluster.fqdn'] }}/api/v1/users/administrator/keys
+      ignore_errors: yes
+    - debug:
+        var: gitea_existing_config_TST.results[].json.data
+
+    - block:
+
+        - name: Register SSH public key
+          ansible.builtin.uri:
+            url: https://git.{{ vapp['metacluster.fqdn'] }}/api/v1/user/keys
+            method: POST
+            headers:
+              Authorization: token {{ gitea_api_token.json.sha1 }}
+            body:
+              key: "{{ gitops_sshkey.public_key }}"
+              read_only: false
+              title: GitOps
+
+        - name: Create organization(s)
+          ansible.builtin.uri:
+            url: https://git.{{ vapp['metacluster.fqdn'] }}/api/v1/orgs
+            method: POST
+            headers:
+              Authorization: token {{ gitea_api_token.json.sha1 }}
+            body: "{{ item }}"
+          loop:
+            - full_name: Meta-cluster
+              description: Meta-cluster configuration items
+              username: mc
+              website: https://git.{{ vapp['metacluster.fqdn'] }}/mc
+              location: '[...]'
+              visibility: public
+            - full_name: Workload-cluster
+              description: Workload-cluster configuration items
+              username: wl
+              website: https://git.{{ vapp['metacluster.fqdn'] }}/wl
+              location: '[...]'
+              visibility: public
+          loop_control:
+            label: "{{ item.full_name }}"
+
+        - name: Create repositories
+          ansible.builtin.uri:
+            url: https://git.{{ vapp['metacluster.fqdn'] }}/api/v1/orgs/{{ item.organization }}/repos
+            method: POST
+            headers:
+              Authorization: token {{ gitea_api_token.json.sha1 }}
+            body: "{{ item.body }}"
+          loop:
+            - organization: mc
+              body:
+                name: GitOps.Config
+                # auto_init: true
+                # default_branch: main
+                description: GitOps manifests
+            - organization: wl
+              body:
+                name: Template.GitOps.Config
+                # auto_init: true
+                # default_branch: main
+                description: GitOps manifests
+          loop_control:
+            label: "{{ item.organization + '/' + item.body.name }}"
+
+        - name: Rebase/Push source gitops repository
+          ansible.builtin.shell:
+            cmd: |
+              git config --local http.sslVerify false
+              git remote set-url origin https://administrator:{{ vapp['guestinfo.rootpw'] | urlencode }}@git.{{ vapp['metacluster.fqdn'] }}/mc/GitOps.Config.git
+              git push
+            chdir: /opt/metacluster/git-repositories/gitops
+
+      when: (gitea_existing_config.json is undefined) or (gitea_existing_config.json.data | length == 0)
+
+  module_defaults:
+    ansible.builtin.uri:
+      validate_certs: no
+      status_code: [200, 201]
+      body_format: json
+
+- block:
+
+    - name: Install argo-cd chart
+      kubernetes.core.helm:
+        name: argo-cd
+        chart_ref: /opt/metacluster/helm-charts/argo-cd
+        release_namespace: argo-cd
+        create_namespace: yes
+        wait: yes
+        kubeconfig: "{{ kubeconfig.path }}"
+        values: "{{ components.argocd.chart_values }}"
+
+    - name: Ensure argo-cd API availability
+      ansible.utils.cli_parse:
+        command: curl -k https://gitops.{{ vapp['metacluster.fqdn'] }}/api/version
+        parser:
+          name: ansible.utils.json
+        set_fact: api_readycheck
+      ignore_errors:
+      until: api_readycheck.Version is defined
+      retries: 3
+      delay: 30
+
+    - name: Generate argo-cd API token
+      ansible.builtin.uri:
+        url: https://gitops.{{ vapp['metacluster.fqdn'] }}/api/v1/session
+        method: POST
+        force_basic_auth: yes
+        body:
+          username: admin
+          password: "{{ vapp['guestinfo.rootpw'] }}"
+      register: argocd_api_token
+
+    # - name: Create umbrella application
+    #   ansible.builtin.template:
+    #
+    - name: Configure metacluster-gitops repository
+      ansible.builtin.template:
+        src: gitrepo.j2
+        dest: /var/lib/rancher/k3s/server/manifests/{{ _template.name }}-manifest.yaml
+        owner: root
+        group: root
+        mode: 0600
+      vars:
+        _template:
+          name: argocd-gitrepo-metacluster
+          namespace: argo-cd
+          uid: "{{ lookup('ansible.builtin.password', '/dev/null length=5 chars=ascii_lowercase,digits seed=inventory_hostname') }}"
+          privatekey: "{{ lookup('ansible.builtin.file', '~/.ssh/git_rsa_id') | indent(4, true) }}"
+      notify:
+        - Apply manifests
+
+    - name: Trigger handlers
+      ansible.builtin.meta: flush_handlers
+
+  module_defaults:
+    ansible.builtin.uri:
+      validate_certs: no
+      status_code: [200, 201]
+      body_format: json
+
+- name: Delete container image tarballs/archives
+  ansible.builtin.file:
+    path: "{{ item }}"
+    state: absent
+  with_fileglob: /opt/metacluster/container-images/*.tar
+
+- name: Cleanup tempfile
+  ansible.builtin.file:
+    path: "{{ kubeconfig.path }}"
+    state: absent
+  when: kubeconfig.path is defined

ansible/roles/firstboot/files/ansible_payload/roles/network/tasks/main.yml

@@ -1,10 +1,12 @@
 - name: Set hostname
   ansible.builtin.hostname:
-    name: "{{ ovfproperties['guestinfo.hostname'] }}"
+    name: "{{ vapp['guestinfo.hostname'] }}"
+
 - name: Create netplan configuration file
   ansible.builtin.template:
     src: netplan.j2
     dest: /etc/netplan/00-installer-config.yaml
+
 - name: Apply netplan configuration
   ansible.builtin.shell:
     cmd: /usr/sbin/netplan apply

ansible/roles/firstboot/files/ansible_payload/roles/network/templates/netplan.j2

@@ -3,8 +3,8 @@ network:
   ethernets:
     ens192:
       addresses:
-      - {{ ovfproperties['guestinfo.ipaddress'] }}/{{ ovfproperties['guestinfo.prefixlength'] }}
-      gateway4: {{ ovfproperties['guestinfo.gateway'] }}
+      - {{ vapp['guestinfo.ipaddress'] }}/{{ vapp['guestinfo.prefixlength'] }}
+      gateway4: {{ vapp['guestinfo.gateway'] }}
       nameservers:
         addresses:
-        - {{ ovfproperties['guestinfo.dnsserver'] }}
+        - {{ vapp['guestinfo.dnsserver'] }}

ansible/roles/firstboot/files/ansible_payload/roles/tty/tasks/main.yml

@@ -0,0 +1,20 @@
+- name: Create folder structure(s)
+  ansible.builtin.file:
+    path: "{{ item }}"
+    state: directory
+  loop:
+    - /opt/firstboot
+
+- name: Save tty script file
+  ansible.builtin.template:
+    src: tty.j2
+    dest: /opt/firstboot/tty.sh
+    owner: root
+    group: root
+    mode: 0700
+
+- name: Create @reboot crontab job
+  ansible.builtin.cron:
+    name: tty.consolemessage
+    special_time: reboot
+    job: /opt/firstboot/tty.sh

ansible/roles/firstboot/files/ansible_payload/roles/users/tasks/main.yml

@@ -1,14 +1,16 @@
 - name: Set root password
   ansible.builtin.user:
     name: root
-    password: "{{ ovfproperties['guestinfo.rootpw'] | password_hash('sha512', 65534 | random(seed=ovfproperties['guestinfo.hostname']) | string) }}"
+    password: "{{ vapp['guestinfo.rootpw'] | password_hash('sha512', 65534 | random(seed=vapp['guestinfo.hostname']) | string) }}"
     generate_ssh_key: yes
     ssh_key_bits: 2048
     ssh_key_file: .ssh/id_rsa
+
 - name: Save root SSH publickey
   ansible.builtin.lineinfile:
     path: /root/.ssh/authorized_keys
-    line: "{{ ovfproperties['guestinfo.rootsshkey'] }}"
+    line: "{{ vapp['guestinfo.rootsshkey'] }}"
+
 - name: Disable SSH password authentication
   ansible.builtin.lineinfile:
     path: /etc/ssh/sshd_config
@@ -16,8 +18,18 @@
     line: "{{ item.line }}"
     state: "{{ item.state }}"
   loop:
-  - { regex: '^#PasswordAuthentication', line: 'PasswordAuthentication no', state: present}
-  - { regex: '^PasswordAuthentication yes', line: 'PasswordAuthentication yes', state: absent}
+  - regex: '^#PasswordAuthentication'
+    line: 'PasswordAuthentication no'
+    state: present
+  - regex: '^PasswordAuthentication yes'
+    line: 'PasswordAuthentication yes'
+    state: absent
+
+- name: Create dedicated SSH keypair
+  community.crypto.openssh_keypair:
+    path: /root/.ssh/git_rsa_id
+  register: gitops_sshkey
+
 - name: Delete 'ubuntu' user
   ansible.builtin.user:
     name: ubuntu

ansible/roles/firstboot/files/ansible_payload/roles/vapp/tasks/main.yml

@@ -2,6 +2,21 @@
   ansible.builtin.shell:
     cmd: /usr/bin/vmtoolsd --cmd "info-get guestinfo.ovfEnv"
   register: ovfenv
+
+- name: Parse XML for MoRef ID
+  community.general.xml:
+    xmlstring: "{{ ovfenv.stdout }}"
+    namespaces:
+      ns: http://schemas.dmtf.org/ovf/environment/1
+      ve: http://www.vmware.com/schema/ovfenv
+    xpath: /ns:Environment
+    content: attribute
+  register: environment_attribute
+
+- name: Store MoRef ID
+  ansible.builtin.set_fact:
+    moref_id: "{{ ((environment_attribute.matches[0].values() | list)[0].values() | list)[1] }}"
+
 - name: Parse XML for vApp properties
   community.general.xml:
     xmlstring: "{{ ovfenv.stdout }}"
@@ -9,13 +24,15 @@
       ns: http://schemas.dmtf.org/ovf/environment/1
     xpath: /ns:Environment/ns:PropertySection/ns:Property
     content: attribute
-  register: ovfenv
+  register: property_section
+
 - name: Assign vApp properties to dictionary
   ansible.builtin.set_fact:
-    ovfproperties: >-
-      {{ ovfproperties | default({}) |
-      combine({((item.values() | list)[0].values() | list)[0]:
-      ((item.values() | list)[0].values() | list)[1]})
+    vapp: >-
+      {{ vapp | default({}) | combine({
+        ((item.values() | list)[0].values() | list)[0]:
+        ((item.values() | list)[0].values() | list)[1]})
       }}
-  loop: "{{ ovfenv.matches }}"
-  no_log: true
+  loop: "{{ property_section.matches }}"
+  loop_control:
+    label: "{{ ((item.values() | list)[0].values() | list)[0] }}"

ansible/roles/firstboot/files/ansible_payload/roles/workloadcluster/tasks/main.yml

@@ -0,0 +1,106 @@
+- block:
+
+    - name: Gather hypervisor details
+      ansible.builtin.shell:
+        cmd: govc ls -L {{ item.moref }} | awk -F/ '{print ${{ item.part }}}'
+      environment:
+        GOVC_INSECURE: '1'
+        GOVC_URL: "{{ vapp['hv.fqdn'] }}"
+        GOVC_USERNAME: "{{ vapp['hv.username'] }}"
+        GOVC_PASSWORD: "{{ vapp['hv.password'] }}"
+      register: govc_inventory
+      loop:
+        - attribute: cluster
+          moref: >-
+            $(govc object.collect -json VirtualMachine:{{ moref_id }} | \
+              jq -r '.[] | select(.Name == "runtime").Val.Host | .Type + ":" + .Value')
+          part: (NF-1)
+        - attribute: datacenter
+          moref: VirtualMachine:{{ moref_id }}
+          part: 2
+        - attribute: datastore
+          moref: >-
+            $(govc object.collect -json VirtualMachine:{{ moref_id }} | \
+              jq -r '.[] | select(.Name == "datastore").Val.ManagedObjectReference | .[].Type + ":" + .[].Value')
+          part: NF
+        - attribute: folder
+          moref: >-
+            $(govc object.collect -json VirtualMachine:{{ moref_id }} | \
+              jq -r '.[] | select(.Name == "parent").Val | .Type + ":" + .Value')
+          part: 0
+        # - attribute: host
+        #   moref: >-
+        #     $(govc object.collect -json VirtualMachine:{{ moref_id }} | \
+        #       jq -r '.[] | select(.Name == "runtime").Val.Host | .Type + ":" + .Value')
+        #   part: NF
+        - attribute: network
+          moref: >-
+            $(govc object.collect -json VirtualMachine:{{ moref_id }} | \
+              jq -r '.[] | select(.Name == "network").Val.ManagedObjectReference | .[].Type + ":" + .[].Value')
+          part: NF
+      loop_control:
+        label: "{{ item.attribute }}"
+
+    - name: Store hypervisor details in dictionary
+      ansible.builtin.set_fact:
+        vcenter_info: "{{ vcenter_info | default({}) | combine({ item.item.attribute : item.stdout }) }}"
+      loop: "{{ govc_inventory.results }}"
+      loop_control:
+        label: "{{ item.item.attribute }}"
+
+- block:
+
+    - name: Check for existing templates on hypervisor
+      community.vmware.vmware_guest_info:
+        name: "{{ (item | basename | split('.'))[:-1] | join('.') }}"
+      register: existing_ova
+      with_fileglob: /opt/workloadcluster/node-templates/*.ova
+      ignore_errors: yes
+
+    - name: Parse OVA files for network mappings
+      ansible.builtin.shell:
+        cmd: govc import.spec -json {{ item }}
+      environment:
+        GOVC_INSECURE: '1'
+        GOVC_URL: "{{ vapp['hv.fqdn'] }}"
+        GOVC_USERNAME: "{{ vapp['hv.username'] }}"
+        GOVC_PASSWORD: "{{ vapp['hv.password'] }}"
+      register: ova_spec
+      when: existing_ova.results[index] is failed
+      with_fileglob: /opt/workloadcluster/node-templates/*.ova
+      loop_control:
+        index_var: index
+
+    - name: Deploy OVA templates on hypervisor
+      community.vmware.vmware_deploy_ovf:
+        cluster: "{{ vcenter_info.cluster }}"
+        datastore: "{{ vcenter_info.datastore }}"
+        folder: "{{ vcenter_info.folder }}"
+        name: "{{ (item | basename | split('.'))[:-1] | join('.') }}"
+        networks: "{u'{{ ova_spec.results[index].stdout | from_json | json_query('NetworkMapping[0].Name') }}':u'{{ vcenter_info.network }}'}"
+        allow_duplicates: no
+        power_on: false
+        ovf: "{{ item }}"
+      register: ova_deploy
+      when: existing_ova.results[index] is failed
+      with_fileglob: /opt/workloadcluster/node-templates/*.ova
+      loop_control:
+        index_var: index
+
+    - name: Mark deployed VM's as templates
+      community.vmware.vmware_guest:
+        name: "{{ item.instance.hw_name }}"
+        is_template: yes
+      when: ova_deploy.results[index] is not skipped
+      loop: "{{ ova_deploy.results }}"
+      loop_control:
+        index_var: index
+        label: "{{ item.item }}"
+
+  module_defaults:
+    group/vmware:
+      hostname: "{{ vapp['hv.fqdn'] }}"
+      validate_certs: no
+      username: "{{ vapp['hv.username'] }}"
+      password: "{{ vapp['hv.password'] }}"
+      datacenter: "{{ vcenter_info.datacenter }}"

ansible/roles/firstboot/files/ansible_payload/templates/gitrepo.j2

@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ _template.name }}-{{ _template.uid }}
+  namespace: {{ _template.namespace }}
+  labels:
+    argocd.argoproj.io/secret-type: repository
+stringData:
+  url: ssh://git@gitea-ssh.gitea.svc.cluster.local/mc/GitOps.Config.git
+  name: {{ _template.name }}
+  insecure: 'true'
+  sshPrivateKey: |
+{{ _template.privatekey }}

ansible/roles/firstboot/files/ansible_payload/templates/ingressroute.j2

@@ -0,0 +1,7 @@
+apiVersion: traefik.containo.us/v1alpha1
+kind: IngressRoute
+metadata:
+  name: {{ _template.name }}
+  namespace: {{ _template.namespace }}
+spec:
+{{ _template.config }}

ansible/roles/firstboot/files/ansible_payload/templates/ingressroutetcp.j2

@@ -0,0 +1,7 @@
+apiVersion: traefik.containo.us/v1alpha1
+kind: IngressRouteTCP
+metadata:
+  name: {{ _template.name }}
+  namespace: {{ _template.namespace }}
+spec:
+{{ _template.config }}

ansible/roles/firstboot/files/ansible_payload/templates/registries.j2

@@ -0,0 +1,25 @@
+mirrors:
+  cr.step.cm:
+    endpoint:
+      - https://registry.{{ vapp['metacluster.fqdn'] }}
+    rewrite:
+      "(.*)": "library/cr.step.sm/$1"
+  docker.io:
+    endpoint:
+      - https://registry.{{ vapp['metacluster.fqdn'] }}
+    rewrite:
+      "(.*)": "library/docker.io/$1"
+  ghcr.io:
+    endpoint:
+      - https://registry.{{ vapp['metacluster.fqdn'] }}
+    rewrite:
+      "(.*)": "library/ghcr.io/$1"
+  quay.io:
+    endpoint:
+      - https://registry.{{ vapp['metacluster.fqdn'] }}
+    rewrite:
+      "(.*)": "library/quay.io/$1"
+configs:
+  registry.{{ vapp['metacluster.fqdn'] }}:
+    tls:
+      insecure_skip_verify: true

ansible/roles/firstboot/files/ansible_payload/templates/tty.j2

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+export TERM=linux
+
+BGRN='\033[1;92m'
+BGRY='\033[1;30m'
+BBLU='\033[1;34m'
+BRED='\033[1;91m'
+BWHI='\033[1;97m'
+CBLA='\033[?16;0;30c' # Hide blinking cursor
+DFLT='\033[0m'        # Reset colour
+LCLR='\033[K'         # Clear to end of line
+PRST='\033[0;0H'      # Reset cursor position
+
+# COMPONENTS=('ca' 'ingress' 'storage' 'registry' 'git' 'gitops')
+COMPONENTS=('ca' 'storage' 'registry' 'git' 'gitops')
+FQDN='{{ vapp['metacluster.fqdn'] }}'
+IPADDRESS='{{ vapp['guestinfo.ipaddress'] }}'
+
+# Waiting to allow boot sequence to finish; crude!
+sleep 30
+clear > /dev/tty1
+
+while /bin/true; do
+  echo -e "${PRST}" > /dev/tty1
+  echo -e "\n\n\t${DFLT}To manage this appliance, please connect to one of the following:${LCLR}\n" > /dev/tty1
+
+  for c in "${COMPONENTS[@]}"; do
+    STATUS=$(curl -ks "https://${c}.${FQDN}" -o /dev/null -w '%{http_code}')
+
+    if [[ "${STATUS}" -eq "200" ]]; then
+      echo -e "\t [${BGRN}+${DFLT}] ${BBLU}https://${c}.${FQDN}${DFLT}${LCLR}" > /dev/tty1
+    else
+      echo -e "\t [${BRED}-${DFLT}] ${BBLU}https://${c}.${FQDN}${DFLT}${LCLR}" > /dev/tty1
+    fi
+  done
+
+  echo -e "\n\t${BGRY}Note that your DNS zone ${DFLT}must have${BGRY} respective records defined,\n\teach pointing to: ${DFLT}${IPADDRESS}${LCLR}" > /dev/tty1
+
+  echo -e "${CBLA}" > /dev/tty1
+  sleep 1
+done

ansible/roles/firstboot/tasks/main.yml

@@ -2,6 +2,7 @@
   ansible.builtin.file:
     path: /opt/firstboot
     state: directory
+
 - name: Create firstboot script file
   ansible.builtin.template:
     src: firstboot.j2
@@ -9,11 +10,13 @@
     owner: root
     group: root
     mode: o+x
+
 - name: Create @reboot crontab job
   ansible.builtin.cron:
     name: firstboot
     special_time: reboot
-    job: "/opt/firstboot/firstboot.sh"
+    job: "/opt/firstboot/firstboot.sh >/dev/tty1 2>&1"
+
 - name: Copy payload folder
   ansible.builtin.copy:
     src: ansible_payload/
@@ -21,6 +24,3 @@
     owner: root
     group: root
     mode: '0644'
-- name: Install ansible-galaxy collection
-  ansible.builtin.shell:
-    cmd: ansible-galaxy collection install community.general

ansible/roles/firstboot/templates/firstboot.j2

@@ -1,4 +1,4 @@
 #!/bin/bash
 
 # Apply firstboot configuration w/ ansible
-/usr/local/bin/ansible-playbook /opt/firstboot/ansible/playbook.yml | tee -a /var/log/firstboot.log > /dev/tty1
+/usr/local/bin/ansible-playbook /opt/firstboot/ansible/playbook.yml | tee -a /var/log/firstboot.log > /dev/tty1 2>&1

ansible/roles/metacluster/tasks/cleanup.yml

@@ -0,0 +1,4 @@
+- name: Zero-out disk
+  ansible.builtin.shell:
+    cmd: nice -n 10 dd bs=1M count=$(df -m . | awk '/[0-9]%/{print $(NF-2)}') if=/dev/zero of=./zero; sync; sync; rm -f ./zero
+    chdir: /opt/metacluster

ansible/roles/metacluster/tasks/components.yml

@@ -0,0 +1,75 @@
+- name: Create folder structure(s)
+  ansible.builtin.file:
+    path: "{{ item }}"
+    state: directory
+  loop:
+    - /opt/metacluster/helm-charts
+    - /opt/metacluster/container-images
+
+- name: Add helm repositories
+  kubernetes.core.helm_repository:
+    name: "{{ item.name }}"
+    repo_url: "{{ item.url }}"
+    state: present
+  loop: "{{ platform.helm_repositories }}"
+
+- name: Fetch helm charts
+  ansible.builtin.command:
+    cmd: helm fetch {{ item.value.helm.chart }} --untar --version {{ item.value.helm.version }}
+    chdir: /opt/metacluster/helm-charts
+  loop: "{{ lookup('ansible.builtin.dict', components) }}"
+  loop_control:
+    label: "{{ item.key }}"
+
+- block:
+
+    - name: Aggregate chart_values into dict
+      ansible.builtin.set_fact:
+        chart_values: "{{ chart_values | default({}) | combine({ (item.key | regex_replace('[^A-Za-z0-9]', '')): { 'chart_values': (item.value.helm.chart_values | from_yaml) } }) }}"
+      when: item.value.helm.chart_values is defined
+      loop: "{{ lookup('ansible.builtin.dict', components) }}"
+      loop_control:
+        label: "{{ item.key }}"
+
+    - name: Write dict to vars_file
+      ansible.builtin.copy:
+        dest: /opt/firstboot/ansible/vars/metacluster.yml
+        content: "{{ { 'components': chart_values } | to_nice_yaml(indent=2, width=4096) }}"
+
+- name: Parse helm charts for container images
+  ansible.builtin.shell:
+    cmd: "{{ item.value.helm.parse_logic }}"
+    chdir: /opt/metacluster/helm-charts/{{ item.key }}
+  register: containerimages
+  loop: "{{ lookup('ansible.builtin.dict', components) }}"
+  loop_control:
+    label: "{{ item.key }}"
+
+- name: Pull and store containerimages
+  ansible.builtin.shell:
+    cmd: >-
+      skopeo copy \
+        --insecure-policy \
+        --retry-times=5 \
+        docker://{{ item }} \
+        docker-archive:./{{ ( item | regex_findall('[^/:]+'))[-2] }}.tar:{{ item }}
+    chdir: /opt/metacluster/container-images
+  loop: "{{ ((containerimages.results | map(attribute='stdout_lines') | flatten) + dependencies.container_images) | unique }}"
+
+# - name: Inject manifests
+#   ansible.builtin.template:
+#     src: "{{ item.type }}.j2"
+#     dest: /var/lib/rancher/k3s/server/manifests/{{ item.name }}-manifest.yaml
+#     owner: root
+#     group: root
+#     mode: 0600
+#   loop: "{{ lookup('ansible.builtin.dict', components) | map(attribute='value.manifests') | list | select('defined') | flatten }}"
+#   loop_control:
+#     label: "{{ item.type + '/' + item.name }}"
+
+- name: Compress tarballs
+  community.general.archive:
+    dest: /opt/metacluster/container-images/image-tarballs.tgz
+    path: /opt/metacluster/container-images/*
+    format: xz
+    remove: yes

ansible/roles/metacluster/tasks/k3s.yml

@@ -0,0 +1,40 @@
+- name: Create folder structure(s)
+  ansible.builtin.file:
+    path: "{{ item }}"
+    state: directory
+  loop:
+    - /var/lib/rancher/k3s/agent/images
+    - /var/lib/rancher/k3s/server/manifests
+    - /opt/metacluster/k3s
+
+- name: Download & install K3s binary
+  ansible.builtin.get_url:
+    url: https://github.com/k3s-io/k3s/releases/download/{{ platform.k3s.version }}/k3s
+    dest: /usr/local/bin/k3s
+    owner: root
+    group: root
+    mode: 0755
+
+- name: Download K3s images tarball
+  ansible.builtin.get_url:
+    url: https://github.com/k3s-io/k3s/releases/download/{{ platform.k3s.version }}/k3s-airgap-images-amd64.tar.gz
+    dest: /var/lib/rancher/k3s/agent/images
+
+- name: Download K3s install script
+  ansible.builtin.get_url:
+    url: https://get.k3s.io
+    dest: /opt/metacluster/k3s/install.sh
+    owner: root
+    group: root
+    mode: 0755
+
+- name: Inject manifests
+  ansible.builtin.template:
+    src: helmchartconfig.j2
+    dest: /var/lib/rancher/k3s/server/manifests/{{ item.name }}-config.yaml
+    owner: root
+    group: root
+    mode: 0600
+  loop: "{{ platform.packaged_components }}"
+  loop_control:
+    label: "{{ item.name }}"

ansible/roles/metacluster/tasks/main.yml

@@ -0,0 +1,11 @@
+- name: Pre-stage K3s components
+  import_tasks: k3s.yml
+
+- name: Pre-stage meta-cluster components
+  import_tasks: components.yml
+
+- name: Pre-stage meta-cluster configuration and workload-cluster components
+  import_tasks: staging.yml
+
+- name: Cleanup
+  import_tasks: cleanup.yml

ansible/roles/metacluster/tasks/staging.yml

@@ -0,0 +1,21 @@
+- name: Create folder structure(s)
+  ansible.builtin.file:
+    path: "{{ item }}"
+    state: directory
+  loop:
+    - /opt/metacluster/git-repositories/gitops
+    - /opt/workloadcluster/node-templates
+
+- name: Clone git repository
+  ansible.builtin.git:
+    repo: "{{ platform.gitops.repository.uri }}"
+    version: "{{ platform.gitops.repository.revision }}"
+    dest: /opt/metacluster/git-repositories/gitops
+
+- name: Download node-template images
+  ansible.builtin.uri:
+    url: "{{ item.url }}"
+    dest: /opt/workloadcluster/node-templates/{{ downstream.node_templates.prefix }}{{ item.name }}
+  loop: "{{ downstream.node_templates.images }}"
+  loop_control:
+    label: "{{ downstream.node_templates.prefix }}{{ item.name }}"

ansible/roles/metacluster/templates/helmchartconfig.j2

@@ -0,0 +1,8 @@
+apiVersion: helm.cattle.io/v1
+kind: HelmChartConfig
+metadata:
+  name: {{ item.name }}
+  namespace: {{ item.namespace }}
+spec:
+  valuesContent: |-
+{{ item.config }}

ansible/roles/os/tasks/ansible.yml

@@ -1,6 +0,0 @@
-- name: Install ansible (w/ dependencies)
-  ansible.builtin.pip:
-    name: "{{ item }}"
-    executable: pip3
-    state: latest
-  loop: "{{ pip_packages }}"

ansible/roles/os/tasks/cloud-init.yml

@@ -3,6 +3,7 @@
     name: cloud-init
     state: absent
     purge: yes
+
 - name: Delete cloud-init files
   ansible.builtin.file:
     path: "{{ item }}"

ansible/roles/os/tasks/main.yml

@@ -15,6 +15,3 @@
 
 - name: Install packages
   import_tasks: packages.yml
-
-- name: Install ansible
-  import_tasks: ansible.yml

ansible/roles/os/tasks/packages.yml

@@ -1,14 +1,23 @@
 - name: Install additional packages
   ansible.builtin.apt:
-    name: "{{ item }}"
+    pkg: "{{ packages.apt }}"
     state: latest
     update_cache: yes
-  loop: "{{ packages }}"
+    install_recommends: no
+
 - name: Upgrade all packages
   ansible.builtin.apt:
-    name: "*"
+    name: '*'
     state: latest
     update_cache: yes
+
+- name: Install additional python packages
+  ansible.builtin.pip:
+    name: "{{ item }}"
+    executable: pip3
+    state: latest
+  loop: "{{ packages.pip }}"
+
 - name: Cleanup
   ansible.builtin.apt:
     autoremove: yes

ansible/roles/os/tasks/snapd.yml

@@ -3,14 +3,17 @@
     name: snapd
     state: absent
     purge: yes
+
 - name: Delete leftover files
   ansible.builtin.file:
     path: /root/snap
     state: absent
+
 - name: Hold snapd package
   ansible.builtin.dpkg_selections:
     name: snapd
     selection: hold
+
 - name: Reload systemd unit configurations
   ansible.builtin.systemd:
     daemon_reload: yes

ansible/roles/os/tasks/tty.yml

@@ -4,8 +4,11 @@
     regexp: "{{ item.regexp }}"
     line: "{{ item.line }}"
   loop:
-  - { regexp: '^#NAutoVTs=', line: 'NAutoVTs=1'}
-  - { regexp: '^#ReserveVT=', line: 'ReserveVT=11'}
+  - regexp: '^#NAutoVTs='
+    line: 'NAutoVTs=1'
+  - regexp: '^#ReserveVT='
+    line: 'ReserveVT=11'
+
 - name: Mask getty@tty1 service
   ansible.builtin.systemd:
     name: getty@tty1

ansible/roles/os/vars/main.yml

@@ -1,11 +1,12 @@
 packages:
-  - jq
-  # (python3-*) Dependency for installation of Ansible
-  - python3-pip
-  - python3-setuptools
-  - python3-wheel
-
-pip_packages:
-  - pip
-  - ansible-core
-  - lxml
+  apt:
+    - jq
+    - python3-pip
+  pip:
+    - ansible-core
+    - jinja2
+    - lxml
+    - markupsafe
+    - pip
+    - setuptools
+    - wheel

ansible/vars/metacluster.yml

@@ -0,0 +1,192 @@
+platform:
+
+  k3s:
+    version: v1.24.1+k3s1
+
+  gitops:
+    repository:
+      uri: https://code.spamasaurus.com/djpbessems/GitOps.MetaCluster.git
+      # revision: v0.1.0
+      revision: HEAD
+
+  packaged_components:
+    - name: traefik
+      namespace: kube-system
+      config: |2
+            additionalArguments:
+              - "--certificatesResolvers.stepca.acme.caserver=https://step-certificates.step-ca.svc.cluster.local/acme/acme/directory"
+              - "--certificatesResolvers.stepca.acme.email=admin"
+              - "--certificatesResolvers.stepca.acme.httpChallenge=true"
+              - "--certificatesResolvers.stepca.acme.httpChallenge.entryPoint=web"
+              - "--certificatesResolvers.stepca.acme.storage=/data/acme.json"
+              - "--certificatesResolvers.stepca.acme.tlsChallenge=true"
+            globalArguments: []
+            ingressRoute:
+              dashboard:
+                enabled: false
+            ports:
+              ssh:
+                port: 8022
+                protocol: TCP
+              web:
+                redirectTo: websecure
+
+  helm_repositories:
+    - name: longhorn
+      url: https://charts.longhorn.io
+    - name: harbor
+      url: https://helm.goharbor.io
+    - name: gitea-charts
+      url: https://dl.gitea.io/charts/
+    - name: argo
+      url: https://argoproj.github.io/argo-helm
+    - name: sealed-secrets
+      url: https://bitnami-labs.github.io/sealed-secrets
+    - name: smallstep
+      url: https://smallstep.github.io/helm-charts/
+
+components:
+
+  longhorn:
+    helm:
+      version: 1.3.0
+      chart: longhorn/longhorn
+      parse_logic: cat values.yaml | yq eval '.. | select(has("repository")) | .repository + ":" + .tag'
+      chart_values: !unsafe |
+        defaultSettings:
+          defaultDataPath: /mnt/blockstorage
+          defaultReplicaCount: 1
+        ingress:
+          enabled: true
+          host: storage.{{ vapp['metacluster.fqdn'] }}
+        persistence:
+          defaultClassReplicaCount: 1
+
+  step-certificates:
+    helm:
+      version: 1.18.2+20220324
+      chart: smallstep/step-certificates
+      parse_logic: helm template . | yq --no-doc eval '.. | .image? | select(.)' | sed '/:/!s/$/:latest/' | sort -u
+      chart_values: !unsafe |
+        inject:
+          secrets:
+            ca_password: "{{ vapp['guestinfo.rootpw'] | b64encode }}"
+            provisioner_password: "{{ vapp['guestinfo.rootpw'] | b64encode }}"
+        ingress:
+          enabled: true
+          hosts:
+            - host: ca.{{ vapp['metacluster.fqdn'] }}
+              paths:
+                - path: /
+                  pathType: Prefix
+        service:
+          targetPort: 9000
+
+  harbor:
+    helm:
+      version: 1.9.1  # (= Harbor v2.5.1)
+      chart: harbor/harbor
+      parse_logic: helm template . | yq --no-doc eval '.. | .image? | select(.)' | sort -u | awk '!/ /'
+      chart_values: !unsafe |
+        expose:
+          ingress:
+            annotations: {}
+            hosts:
+              core: registry.{{ vapp['metacluster.fqdn'] }}
+          tls:
+            certSource: none
+            enabled: false
+        externalURL: https://registry.{{ vapp['metacluster.fqdn'] }}
+        harborAdminPassword: "{{ vapp['guestinfo.rootpw'] }}"
+        notary:
+          enabled: false
+
+  gitea:
+    helm:
+      version: v5.0.9 # (= Gitea v1.16.8)
+      chart: gitea-charts/gitea
+      parse_logic: helm template . | yq --no-doc eval '.. | .image? | select(.)' | sort -u | sed '/:/!s/$/:latest/'
+      chart_values: !unsafe |
+        config:
+          offlineMode: true
+        gitea:
+          admin:
+            username: administrator
+            password: "{{ vapp['guestinfo.rootpw'] }}"
+            email: admin@{{ vapp['metacluster.fqdn'] }}
+        image:
+          pullPolicy: IfNotPresent
+        ingress:
+          enabled: true
+          hosts:
+            - host: git.{{ vapp['metacluster.fqdn'] }}
+              paths:
+                - path: /
+                  pathType: Prefix
+        service:
+          ssh:
+            type: ClusterIP
+            port: 22
+            clusterIP:
+
+  argo-cd:
+    helm:
+      version: 4.9.7  # (= ArgoCD v2.4.2)
+      chart: argo/argo-cd
+      parse_logic: helm template . | yq --no-doc eval '.. | .image? | select(.)' | sort -u | awk '!/ /'
+      chart_values: !unsafe |
+        server:
+          extraArgs:
+            - --insecure
+          ingress:
+            enabled: true
+            hosts:
+              - gitops.{{ vapp['metacluster.fqdn'] }}
+        configs:
+          secret:
+            argocdServerAdminPassword: "{{ vapp['guestinfo.rootpw'] | password_hash('bcrypt') }}"
+
+  sealed-secrets:
+    helm:
+      version: 2.4.0  # (= SealedSecrets v0.18.1)
+      chart: sealed-secrets/sealed-secrets
+      parse_logic: helm template . | yq --no-doc eval '.. | .image? | select(.)' | sort -u | awk '!/ /'
+
+dependencies:
+
+  ansible_galaxy_collections:
+    - ansible.posix
+    - ansible.utils
+    - community.crypto
+    - community.general
+    - community.vmware
+    - kubernetes.core
+
+  container_images:
+    - vmware/powerclicore:12.7
+
+  static_binaries:
+    - filename: govc
+      url: https://github.com/vmware/govmomi/releases/download/v0.29.0/govc_Linux_x86_64.tar.gz
+      archive: compressed
+    - filename: helm
+      url: https://get.helm.sh/helm-v3.9.0-linux-amd64.tar.gz
+      archive: compressed
+      extra_opts: --strip-components=1
+    - filename: skopeo
+      url: https://code.spamasaurus.com/api/packages/djpbessems/generic/skopeo/v1.9.1/skopeo
+    - filename: step
+      url: https://dl.step.sm/gh-release/cli/gh-release-header/v0.21.0/step_linux_0.21.0_amd64.tar.gz
+      archive: compressed
+      extra_opts: --strip-components=2
+    - filename: yq
+      url: http://github.com/mikefarah/yq/releases/download/v4.25.3/yq_linux_amd64
+
+  packages:
+    apt:
+      - lvm2
+    pip:
+      - jmespath
+      - kubernetes
+      - passlib
+      - pyvmomi

ansible/vars/workloadcluster.yml

@@ -0,0 +1,7 @@
+downstream:
+
+  node_templates:
+    prefix: NodeTmpl_
+    images:
+      - url: https://stable.release.flatcar-linux.net/amd64-usr/3227.2.0/flatcar_production_vmware_ova.ova
+        name: flatcar.ova

packer/iso.auto.pkrvars.hcl

@@ -1,4 +1,4 @@
-iso_url      = "sn.itch.fyi/Repository/iso/Canonical/Ubuntu%20Server%2020.04/ubuntu-20.04.4-live-server-amd64.iso"
-iso_checksum = "sha256:28CCDB56450E643BAD03BB7BCF7507CE3D8D90E8BF09E38F6BD9AC298A98EAAD"
+iso_url      = "sn.itch.fyi/Repository/iso/Canonical/Ubuntu%20Server%2022.04/ubuntu-22.04-live-server-amd64.iso"
+iso_checksum = "sha256:84AEAF7823C8C61BAA0AE862D0A06B03409394800000B3235854A6B38EB4856F"
 // iso_url      = "sn.itch.fyi/Repository/iso/Canonical/Ubuntu%20Server%2020.04/ubuntu-20.04.2-live-server-amd64.iso"
 // iso_checksum = "sha256:D1F2BF834BBE9BB43FAF16F9BE992A6F3935E65BE0EDECE1DEE2AA6EB1767423"

packer/k8sbootstrap.pkr.hcl

@@ -3,7 +3,7 @@ packer {
   }
 }
 
-source "vsphere-iso" "ubuntuserver" {
+source "vsphere-iso" "k8sbootstrap" {
   vcenter_server          = var.vcenter_server
   username                = var.vsphere_username
   password                = var.vsphere_password
@@ -20,9 +20,9 @@ source "vsphere-iso" "ubuntuserver" {
 
   boot_order              = "disk,cdrom"
   boot_command            = [
-    "<enter><wait2><enter><wait><f6><esc><wait>",
-    " autoinstall<wait2> ds=nocloud;",
-    "<wait><enter>"
+    "e<down><down><down><end>",
+    " autoinstall ds=nocloud;",
+    "<F10>"
   ]
   boot_wait               = "2s"
 
@@ -34,22 +34,22 @@ source "vsphere-iso" "ubuntuserver" {
   ssh_pty                 = true
 
   CPUs                    = 2
-  RAM                     = 4096
+  RAM                     = 8192
 
   network_adapters {
     network               = var.vsphere_network
     network_card          = "vmxnet3"
   }
   storage {
-    disk_size             = 20480
+    disk_size             = 76800
     disk_thin_provisioned = true
   }
   disk_controller_type    = ["pvscsi"]
   usb_controller          = ["xhci"]
 
   cd_files                = [
-    "packer/preseed/UbuntuServer20.04/user-data",
-    "packer/preseed/UbuntuServer20.04/meta-data"
+    "packer/preseed/UbuntuServer22.04/user-data",
+    "packer/preseed/UbuntuServer22.04/meta-data"
   ]
   cd_label                = "cidata"
   iso_url                 = local.iso_authenticatedurl
@@ -60,18 +60,18 @@ source "vsphere-iso" "ubuntuserver" {
 
   export {
     images                = false
-    output_directory      = "/scratch/ubuntuserver"
+    output_directory      = "/scratch/k8sbootstrap"
   }
   remove_cdrom            = true
 }
 
 build {
   sources = [
-    "source.vsphere-iso.ubuntuserver"
+    "source.vsphere-iso.k8sbootstrap"
   ]
 
   provisioner "ansible" {
-    only             = ["vsphere-iso.ubuntuserver"]
+    pause_before     = "2m30s"
 
     playbook_file    = "ansible/playbook.yml"
     user             = "ubuntu"
@@ -80,21 +80,22 @@ build {
     ]
     use_proxy        = "false"
     extra_arguments  = [
-      "--extra-vars", "ansible_ssh_pass=${var.ssh_password}"
+      "--extra-vars", "ansible_ssh_pass=${var.ssh_password}",
+      "--extra-vars", "repo_username=${var.repo_username}",
+      "--extra-vars", "repo_password=${var.repo_password}"
     ]
   }
 
   post-processor "shell-local" {
-    only   = ["vsphere-iso.ubuntuserver"]
     inline = [
       "pwsh -command \"& scripts/Update-OvfConfiguration.ps1 \\",
-      " -OVFFile '/scratch/ubuntuserver/${var.vm_guestos}-${var.vm_name}.ovf' \\",
+      " -OVFFile '/scratch/k8sbootstrap/${var.vm_guestos}-${var.vm_name}.ovf' \\",
       " -Parameter @{'appliance.name'='${var.vm_guestos}';'appliance.version'='${var.vm_name}'}\"",
       "pwsh -file scripts/Update-Manifest.ps1 \\",
-      " -ManifestFileName '/scratch/ubuntuserver/${var.vm_guestos}-${var.vm_name}.mf'",
+      " -ManifestFileName '/scratch/k8sbootstrap/${var.vm_guestos}-${var.vm_name}.mf'",
       "ovftool --acceptAllEulas --allowExtraConfig --overwrite \\",
-      " '/scratch/ubuntuserver/${var.vm_guestos}-${var.vm_name}.ovf' \\",
-      " /output/Ubuntu-Server-20.04.ova"
+      " '/scratch/k8sbootstrap/${var.vm_guestos}-${var.vm_name}.ovf' \\",
+      " /output/Kubernetes.Bootstrap.Appliance.ova"
     ]
   }
-}
+}

packer/preseed/UbuntuServer22.04/user-data

@@ -11,6 +11,7 @@ autoinstall:
       ethernets:
         ens192:
           dhcp4: true
+          dhcp-identifier: mac
   storage:
     layout:
       name: direct

packer/vsphere.auto.pkrvars.hcl

@@ -1,9 +1,9 @@
 vcenter_server         = "bv11-vc.bessems.lan"
 vsphere_username       = "administrator@vsphere.local"
 vsphere_datacenter     = "DeSchakel"
-vsphere_cluster        = "Cluster.Legacy"
-vsphere_host           = "bv11-esx.bessems.lan"
-vsphere_datastore      = "ESX00.SSD01"
+vsphere_cluster        = "Cluster.01"
+vsphere_host           = "bv11-esx02.bessems.lan"
+vsphere_datastore      = "NAS01.RAID5"
 vsphere_folder         = "/Packer"
 vsphere_templatefolder = "/Templates"
-vsphere_network        = "LAN"
+vsphere_network        = "LAN"

scripts/Update-OvfConfiguration.yml

@@ -1,35 +1,60 @@
 DeploymentConfigurations:
+
 - Id: small
-  Label: 'Ubuntu Server 20.04 [SMALL: 1 vCPU/2GB RAM]'
-  Description: Ubuntu Server 20.04.x
+  Label: 'Ubuntu Server 22.04 [SMALL: 2 vCPU/4GB RAM]'
+  Description: Ubuntu Server 22.04.x
   Size:
-    CPU: 1
-    Memory: 2048
-- Id: large
-  Label: 'Ubuntu Server 20.04 [LARGE: 4 vCPU/8GB RAM]'
-  Description: Ubuntu Server 20.04.x
+    CPU: 2
+    Memory: 4096
+
+- Id: medium
+  Label: 'Ubuntu Server 22.04 [MEDIUM: 4 vCPU/8GB RAM]'
+  Description: Ubuntu Server 22.04.x
   Size:
     CPU: 4
     Memory: 8192
-DynamicDisks: []
+
+DynamicDisks:
+
+- Description: Longhorn persistent storage
+  UnitSize: GB
+  Constraints:
+    Minimum: 100
+    Maximum: ''
+
 PropertyCategories:
+
 # - Name: 0) Deployment information
 #   ProductProperties:
 #   - Key: deployment.type
 #     Type: string
 #     Value:
 #     - small
-#     - large
+#     - medium
 #     UserConfigurable: false
-- Name: 1) Operating System
+
+- Name: 1) Kubernetes
   ProductProperties:
+
+  - Key: metacluster.fqdn
+    Type: string(1..)
+    Label: Appliance FQDN*
+    Description: 'Respective subdomains will be available for each component (e.g. storage.example.org); this address should already be configured as a wildcard record within your DNS zone.'
+    DefaultValue: 'meta.k8s.cluster'
+    Configurations: '*'
+    UserConfigurable: true
+
+- Name: 2) Operating System
+  ProductProperties:
+
   - Key: guestinfo.hostname
     Type: string(1..15)
     Label: Hostname*
-    Description: '(max length: 15 characters)'
-    DefaultValue: ''
+    Description: ''
+    DefaultValue: 'META-NODE-{{ hostname.suffix }}'
     Configurations: '*'
     UserConfigurable: true
+
   - Key: guestinfo.rootpw
     Type: password(7..)
     Label: Local root password*
@@ -37,6 +62,7 @@ PropertyCategories:
     DefaultValue: ''
     Configurations: '*'
     UserConfigurable: true
+
   - Key: guestinfo.rootsshkey
     Type: password(1..)
     Label: Local root SSH public key*
@@ -44,15 +70,10 @@ PropertyCategories:
     DefaultValue: ''
     Configurations: '*'
     UserConfigurable: true
-  - Key: guestinfo.ntpserver
-    Type: string(1..)
-    Label: Time server*
-    Description: A comma-separated list of timeservers
-    DefaultValue: 0.pool.ntp.org,1.pool.ntp.org,2.pool.ntp.org
-    Configurations: '*'
-    UserConfigurable: true
-- Name: 2) Networking
+
+- Name: 3) Networking
   ProductProperties:
+
   - Key: guestinfo.ipaddress
     Type: ip
     Label: IP Address*
@@ -60,6 +81,7 @@ PropertyCategories:
     DefaultValue: ''
     Configurations: '*'
     UserConfigurable: true
+
   - Key: guestinfo.prefixlength
     Type: int(8..32)
     Label: Subnet prefix length*
@@ -67,6 +89,7 @@ PropertyCategories:
     DefaultValue: '24'
     Configurations: '*'
     UserConfigurable: true
+
   - Key: guestinfo.dnsserver
     Type: ip
     Label: DNS server*
@@ -74,6 +97,7 @@ PropertyCategories:
     DefaultValue: ''
     Configurations: '*'
     UserConfigurable: true
+
   - Key: guestinfo.gateway
     Type: ip
     Label: Gateway*
@@ -81,6 +105,42 @@ PropertyCategories:
     DefaultValue: ''
     Configurations: '*'
     UserConfigurable: true
+
+  - Key: guestinfo.ntpserver
+    Type: string(1..)
+    Label: Time server*
+    Description: A comma-separated list of timeservers
+    DefaultValue: 0.pool.ntp.org,1.pool.ntp.org,2.pool.ntp.org
+    Configurations: '*'
+    UserConfigurable: true
+
+- Name: 4) Hypervisor
+  ProductProperties:
+
+  - Key: hv.fqdn
+    Type: string(1..)
+    Label: vCenter FQDN/IP-address*
+    Description: The address of the vCenter instance which this bootstrap appliance will interact with for provisioning new VM's.
+    DefaultValue: ''
+    Configurations: '*'
+    UserConfigurable: true
+
+  - Key: hv.username
+    Type: string(1..)
+    Label: vCenter username*
+    Description: The username which this bootstrap appliance will authenticate with to the vCenter instance.
+    DefaultValue: ''
+    Configurations: '*'
+    UserConfigurable: true
+
+  - Key: hv.password
+    Type: password(1..)
+    Label: vCenter password ...*
+    Description: The password which this bootstrap appliance will authenticate with to the vCenter instance.
+    DefaultValue: ''
+    Configurations: '*'
+    UserConfigurable: true
+
 AdvancedOptions:
 - Key: appliance.name
   Value: "{{ appliance.name }}"
@@ -91,6 +151,9 @@ AdvancedOptions:
 
 ---
 Variables:
+- Name: hostname.suffix
+  Expression: |
+    (-join ((48..57) + (97..122) | Get-Random -Count 5 | % {[char]$_})).ToUpper()
 - Name: appliance.name
   Expression: |
     $Parameter['appliance.name']