fix: Incorrect indentation causing malformed PEM file

fix: Refactor kustomize templates
feat:Explicitly configure NTP servers (WIP)
2023-07-07 10:29:57 +02:00 · 2023-07-06 13:00:08 +02:00 · 2023-06-03 10:07:09 +02:00 · 2023-06-03 10:06:33 +02:00
11 changed files with 66 additions and 141 deletions
--- a/.drone.yml
+++ b/.drone.yml
@ -10,11 +10,6 @@ volumes:
  claim:
    name: flexvolsmb-drone-scratch

-trigger:
-  event:
-    exclude:
-    - tag
-
 steps:
 - name: Debugging information
  image: bv11-cr01.bessems.eu/library/packer-extended
@ -26,6 +21,8 @@ steps:
  - yamllint --version

 - name: Linting
+  depends_on:
+  - Debugging information
  image: bv11-cr01.bessems.eu/library/packer-extended
  pull: always
  commands:
@ -35,48 +32,9 @@ steps:
      packer/preseed/UbuntuServer22.04/user-data \
      scripts

- name: Semantic Release (Dry-run)
-  image: bv11-cr01.bessems.eu/proxy/library/node:20-slim
-  pull: always
-  commands:
-  - |
-    apt-get update
-  - |
-    apt-get install -y --no-install-recommends \
-      curl \
-      git-core \
-      jq \
-      ca-certificates
-  - |
-    curl -L https://api.github.com/repos/mikefarah/yq/releases/latest | \
-      jq -r '.assets[] | select(.name | endswith("yq_linux_amd64")) | .browser_download_url' | \
-      xargs -I {} curl -L -o /bin/yq {} && \
-    chmod +x /bin/yq
-  - |
-    npm install \
-      semantic-release \
-      @semantic-release/commit-analyzer \
-      @semantic-release/exec \
-  - |
-    export K8S_VERSION=$(yq '.components.clusterapi.workload.version.k8s' < ./ansible/vars/metacluster.yml)
-    export GIT_CREDENTIALS=$${GIT_USERNAME}:$${GIT_APIKEY}
-  - |
-    npx semantic-release \
-      --package @semantic-release/exec \
-      --package semantic-release \
-      --branches ${DRONE_BRANCH} \
-      --tag-format "K8s_$${K8S_VERSION}-v\$${version}" \
-      --dry-run \
-      --plugins @semantic-release/commit-analyzer,@semantic-release/exec \
-      --analyzeCommits @semantic-release/commit-analyzer \
-      --verifyRelease @semantic-release/exec \
-      --verifyReleaseCmd 'echo "$${nextRelease.version}" > .version'
-  environment:
-    GIT_APIKEY:
-      from_secret: git_apikey
-    GIT_USERNAME: djpbessems
-
 - name: Install Ansible Galaxy collections
+  depends_on:
+  - Linting
  image: bv11-cr01.bessems.eu/library/packer-extended
  pull: always
  commands:
@ -84,8 +42,13 @@ steps:
    ansible-galaxy collection install \
      -r ansible/requirements.yml \
      -p ./ansible/collections
+  volumes:
+  - name: scratch
+    path: /scratch

 - name: Kubernetes Bootstrap Appliance
+  depends_on:
+  - Install Ansible Galaxy collections
  image: bv11-cr01.bessems.eu/library/packer-extended
  pull: always
  commands:
@ -94,7 +57,6 @@ steps:
      packer/preseed/UbuntuServer22.04/user-data
  - |
    export K8S_VERSION=$(yq '.components.clusterapi.workload.version.k8s' < ./ansible/vars/metacluster.yml)
-    export APPLIANCE_VERSION=$(cat .version)
  - |
    packer init -upgrade \
      ./packer
@ -109,7 +71,6 @@ steps:
      -var ssh_password=$${SSH_PASSWORD} \
      -var vsphere_password=$${VSPHERE_PASSWORD} \
      -var k8s_version=$K8S_VERSION \
-      -var appliance_version=$APPLIANCE_VERSION \
      ./packer
  - |
    packer build \
@ -123,7 +84,6 @@ steps:
      -var ssh_password=$${SSH_PASSWORD} \
      -var vsphere_password=$${VSPHERE_PASSWORD} \
      -var k8s_version=$K8S_VERSION \
-      -var appliance_version=$APPLIANCE_VERSION \
      ./packer
  environment:
    DOCKER_USERNAME:
@ -146,6 +106,8 @@ steps:
    path: /scratch

 - name: Kubernetes Upgrade Appliance
+  depends_on:
+  - Install Ansible Galaxy collections
  image: bv11-cr01.bessems.eu/library/packer-extended
  pull: alwaysquery(
  commands:
@ -154,7 +116,6 @@ steps:
      packer/preseed/UbuntuServer22.04/user-data
  - |
    export K8S_VERSION=$(yq '.components.clusterapi.workload.version.k8s' < ./ansible/vars/metacluster.yml)
-    export APPLIANCE_VERSION=$(cat .version)
  - |
    packer init -upgrade \
      ./packer
@ -169,7 +130,6 @@ steps:
      -var ssh_password=$${SSH_PASSWORD} \
      -var vsphere_password=$${VSPHERE_PASSWORD} \
      -var k8s_version=$K8S_VERSION \
-      -var appliance_version=$APPLIANCE_VERSION \
      ./packer
  - |
    packer build \
@ -183,7 +143,6 @@ steps:
      -var ssh_password=$${SSH_PASSWORD} \
      -var vsphere_password=$${VSPHERE_PASSWORD} \
      -var k8s_version=$K8S_VERSION \
-      -var appliance_version=$APPLIANCE_VERSION \
      ./packer
  environment:
    DOCKER_USERNAME:
@ -206,6 +165,9 @@ steps:
    path: /scratch

 - name: Remove temporary resources
+  depends_on:
+  - Kubernetes Bootstrap Appliance
+  - Kubernetes Upgrade Appliance
  image: bv11-cr01.bessems.eu/library/packer-extended
  commands:
  - |
--- a/ansible/roles/assets/tasks/manifests.yml
+++ b/ansible/roles/assets/tasks/manifests.yml
@ -16,16 +16,14 @@
            { 'components': (
              metacluster_chartvalues |
              combine({ 'clusterapi': components.clusterapi }) |
-              combine({ 'kubevip'   : components.kubevip }) ),
-              'appliance': {
-                'version': (applianceversion)
-              }
+              combine({ 'kubevip'   : components.kubevip }) )
            } | to_nice_yaml(indent=2, width=4096)
          }}

    - name: Aggregate chart_values into dict
      ansible.builtin.set_fact:
        workloadcluster_chartvalues: "{{ workloadcluster_chartvalues | default({}) | combine({ item.key: { 'chart_values': (item.value.chart_values | default('') | from_yaml) } }) }}"
+      # when: item.value.chart_values is defined
      loop: "{{ query('ansible.builtin.dict', downstream.helm_charts) }}"
      loop_control:
        label: "{{ item.key }}"
--- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/filter_plugins/netaddr.py
+++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/filter_plugins/netaddr.py
@ -0,0 +1,14 @@
+import netaddr
+
+def netaddr_iter_iprange(ip_start, ip_end):
+    return [str(ip) for ip in netaddr.iter_iprange(ip_start, ip_end)]
+
+class FilterModule(object):
+        ''' Ansible filter. Interface to netaddr methods.
+            https://pypi.org/project/netaddr/
+        '''
+
+        def filters(self):
+            return {
+                'netaddr_iter_iprange': netaddr_iter_iprange
+            }
--- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/git.yml
+++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/git.yml
@ -55,6 +55,7 @@
        force_basic_auth: yes
        body:
          name: token_init_{{ lookup('password', '/dev/null length=5 chars=ascii_letters,digits') }}
+          scopes: ["write:public_key","write:org"]
      register: gitea_api_token

    - name: Retrieve existing gitea configuration
--- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/json-server.yml
+++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/json-server.yml
@ -1,27 +0,0 @@
- block:
-    - name: Install json-server chart
-      kubernetes.core.helm:
-        name: json-server
-        chart_ref: /opt/metacluster/helm-charts/json-server
-        release_namespace: json-server
-        create_namespace: true
-        wait: false
-        kubeconfig: "{{ kubeconfig.path }}"
-        values: "{{ components['json-server'].chart_values }}"
-
-    - name: Ensure json-server API availability
-      ansible.builtin.uri:
-        url: https://version.{{ vapp['metacluster.fqdn'] }}/healthz
-        method: GET
-      register: api_readycheck
-      until:
-        - api_readycheck.json.status is defined
-        - api_readycheck.json.status == 'running'
-      retries: "{{ playbook.retries }}"
-      delay: "{{ (storage_benchmark | int) * (playbook.delay.long | int) }}"
-
-  module_defaults:
-    ansible.builtin.uri:
-      validate_certs: no
-      status_code: [200, 201]
-      body_format: json
--- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/main.yml
+++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/roles/metacluster/tasks/main.yml
@ -1,6 +1,5 @@
 - import_tasks: init.yml
 - import_tasks: k3s.yml
- import_tasks: json-server.yml
 - import_tasks: assets.yml
 - import_tasks: kube-vip.yml
 - import_tasks: storage.yml
--- a/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/kustomization.cluster-template.j2
+++ b/ansible/roles/firstboot/files/ansible_payload/bootstrap/templates/kustomization.cluster-template.j2
@ -42,6 +42,11 @@ patches:
      kubeadmConfigSpec:
        clusterConfiguration:
          imageRepository: registry.{{ _template.network.fqdn }}/kubeadm
+        ntp:
+          enabled: true
+          servers:
+            - 0.nl.pool.ntp.org
+            - 1.nl.pool.ntp.org
 - patch: |-
    apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
    kind: KubeadmConfigTemplate
@ -53,6 +58,11 @@ patches:
        spec:
          clusterConfiguration:
            imageRepository: registry.{{ _template.network.fqdn }}/kubeadm
+          ntp:
+            enabled: true
+            servers:
+              - 0.nl.pool.ntp.org
+              - 1.nl.pool.ntp.org
 - patch: |-
    apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
    kind: KubeadmConfigTemplate
--- a/ansible/vars/metacluster.yml
+++ b/ansible/vars/metacluster.yml
@ -1,7 +1,7 @@
 platform:

  k3s:
-    version: v1.27.1+k3s1
+    version: v1.26.5+k3s1

  packaged_components:
    - name: traefik
@ -51,14 +51,12 @@ platform:
      url: https://prometheus-community.github.io/helm-charts
    - name: smallstep
      url: https://smallstep.github.io/helm-charts/
-    - name: spamasaurus
-      url: https://code.spamasaurus.com/api/packages/djpbessems/helm

 components:

  argo-cd:
    helm:
-      version: 5.27.4  # (= ArgoCD v2.6.7)
+      version: 5.34.6  # (= ArgoCD v2.7.3)
      chart: argo/argo-cd
      parse_logic: helm template . | yq --no-doc eval '.. | .image? | select(.)' | sort -u | awk '!/ /'
      chart_values: !unsafe |
@ -101,7 +99,7 @@ components:

  cert-manager:
    helm:
-      version: 1.11.0
+      version: 1.12.1
      chart: jetstack/cert-manager
      parse_logic: helm template . | yq --no-doc eval '.. | .image? | select(.)' | sort -u | awk '!/ /'
      # chart_values: !unsafe |
@ -111,19 +109,19 @@ components:
    management:
      version:
        # Must match the version referenced at `dependencies.static_binaries[.filename==clusterctl].url`
-        base: v1.4.0
+        base: v1.4.1
        # Must match the version referenced at `components.cert-manager.helm.version`
-        cert_manager: v1.11.0
-        infrastructure_vsphere: v1.6.0
+        cert_manager: v1.11.1
+        infrastructure_vsphere: v1.6.1
        ipam_incluster: v0.1.0-alpha.2
        # Refer to `https://console.cloud.google.com/gcr/images/cloud-provider-vsphere/GLOBAL/cpi/release/manager` for available tags
        cpi_vsphere: v1.26.2
    workload:
      version:
-        calico: v3.25.0
-        k8s: v1.27.1
+        calico: v3.26.0
+        k8s: v1.26.5
      node_template:
-        url: https://{{ repo_username }}:{{ repo_password }}@sn.itch.fyi/Repository/rel/ubuntu-2204-kube-v1.27.1.ova
+        url: https://{{ repo_username }}:{{ repo_password }}@sn.itch.fyi/Repository/rel/ubuntu-2204-kube-v1.26.5.ova

  # dex:
  #   helm:
@ -175,7 +173,7 @@ components:

  gitea:
    helm:
-      version: v7.0.2 # (= Gitea v1.18.3)
+      version: v8.3.0 # (= Gitea v1.19.3)
      chart: gitea-charts/gitea
      parse_logic: helm template . | yq --no-doc eval '.. | .image? | select(.)' | sort -u | sed '/:/!s/$/:latest/'
      chart_values: !unsafe |
@ -206,7 +204,7 @@ components:

  harbor:
    helm:
-      version: 1.11.0  # (= Harbor v2.7.0)
+      version: 1.12.1  # (= Harbor v2.8.1)
      chart: harbor/harbor
      parse_logic: helm template . | yq --no-doc eval '.. | .image? | select(.)' | sort -u | awk '!/ /'
      chart_values: !unsafe |
@ -227,34 +225,6 @@ components:
            registry:
              size: 25Gi

-  json-server:
-    helm:
-      version: v0.5.1
-      chart: spamasaurus/json-server
-      parse_logic: helm template . | yq --no-doc eval '.. | .image? | select(.)' | sort -u | awk '!/ /'
-      chart_values: !unsafe |
-        ingress:
-          enabled: true
-          hosts:
-            - host: version.{{ vapp['metacluster.fqdn'] }}
-              paths:
-                - path: /
-                  pathType: Prefix
-        seedData:
-          configInline: |
-            {
-              "appliance": { "version": "{{ appliance.version }}" },
-              "components": [
-                { "app": "argo-cd", "version": "{{ components[argo-cd].helm.version }}" },
-                { "app": "cert-manager", "version": "{{ components[cert-manager].helm.version }}" },
-                { "app": "cluster-api",
-                  "management": { "foo": "bar" },
-                  "workload": { "foo": "bar" }
-                }
-              ],
-              "healthz": { "status": "running" }
-            }
-
  # keycloakx:
  #   helm:
  #     version: 2.1.1  # (= Keycloak 20.0.3)
@ -289,7 +259,7 @@ components:

  kube-prometheus-stack:
    helm:
-      version: 45.2.0
+      version: 46.5.0  # (= Prometheus version v0.65.1)
      chart: prometheus-community/kube-prometheus-stack
      parse_logic: helm template . | yq --no-doc eval '.. | .image? | select(.)' | sort -u | awk '!/ /'
      chart_values: !unsafe |
@ -300,11 +270,11 @@ components:

  kubevip:
    # Must match the version referenced at `dependencies.container_images`
-    version: v0.5.8
+    version: v0.6.0

  longhorn:
    helm:
-      version: 1.4.1
+      version: 1.4.2
      chart: longhorn/longhorn
      parse_logic: cat values.yaml | yq eval '.. | select(has("repository")) | .repository + ":" + .tag'
      chart_values: !unsafe |
@ -320,7 +290,7 @@ components:

  step-certificates:
    helm:
-      version: 1.23.0
+      version: 1.23.2+5  # (= step-ca v0.23.2)
      chart: smallstep/step-certificates
      parse_logic: helm template . | yq --no-doc eval '.. | .image? | select(.)' | sed '/:/!s/$/:latest/' | sort -u
      chart_values: !unsafe |
@ -350,7 +320,7 @@ dependencies:
  container_images:
    # This should match the image tag referenced at `platform.packaged_components[.name==traefik].config`
    - busybox:1
-    - ghcr.io/kube-vip/kube-vip:v0.5.8
+    - ghcr.io/kube-vip/kube-vip:v0.6.0
    # The following list is generated by running the following commands:
    #   $ clusterctl init -i vsphere:<version> [...]
    #   $ clusterctl generate cluster <name> [...] | yq eval '.data.data' | yq --no-doc eval '.. | .image? | select(.)' | sort -u
@ -364,25 +334,25 @@ dependencies:

  static_binaries:
    - filename: clusterctl
-      url: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.4.0/clusterctl-linux-amd64
+      url: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.4.1/clusterctl-linux-amd64
    - filename: govc
-      url: https://github.com/vmware/govmomi/releases/download/v0.29.0/govc_Linux_x86_64.tar.gz
+      url: https://github.com/vmware/govmomi/releases/download/v0.30.4/govc_Linux_x86_64.tar.gz
      archive: compressed
    - filename: helm
-      url: https://get.helm.sh/helm-v3.10.2-linux-amd64.tar.gz
+      url: https://get.helm.sh/helm-v3.12.0-linux-amd64.tar.gz
      archive: compressed
      extra_opts: --strip-components=1
    - filename: kubectl-slice
-      url: https://github.com/patrickdappollonio/kubectl-slice/releases/download/v1.2.5/kubectl-slice_linux_x86_64.tar.gz
+      url: https://github.com/patrickdappollonio/kubectl-slice/releases/download/v1.2.6/kubectl-slice_linux_x86_64.tar.gz
      archive: compressed
    - filename: skopeo
      url: https://code.spamasaurus.com/api/packages/djpbessems/generic/skopeo/v1.12.0/skopeo_linux_amd64
    - filename: step
-      url: https://dl.step.sm/gh-release/cli/gh-release-header/v0.23.0/step_linux_0.23.0_amd64.tar.gz
+      url: https://dl.step.sm/gh-release/cli/gh-release-header/v0.23.2/step_linux_0.23.2_amd64.tar.gz
      archive: compressed
      extra_opts: --strip-components=2
    - filename: yq
-      url: http://github.com/mikefarah/yq/releases/download/v4.30.5/yq_linux_amd64
+      url: http://github.com/mikefarah/yq/releases/download/v4.34.1/yq_linux_amd64

  packages:
    apt:
--- a/ansible/vars/workloadcluster.yml
+++ b/ansible/vars/workloadcluster.yml
@ -9,7 +9,7 @@ downstream:
  helm_charts:

    longhorn:
-      version: 1.4.1
+      version: 1.4.2
      chart: longhorn/longhorn
      namespace: longhorn-system
      parse_logic: cat values.yaml | yq eval '.. | select(has("repository")) | .repository + ":" + .tag'
@ -19,7 +19,7 @@ downstream:
          defaultDataPath: /mnt/blockstorage

    sealed-secrets:
-      version: 2.8.1  # (= Sealed Secrets v0.20.2)
+      version: 2.9.0  # (= Sealed Secrets v0.21.0)
      chart: sealed-secrets/sealed-secrets
      namespace: sealed-secrets
      parse_logic: helm template . | yq --no-doc eval '.. | .image? | select(.)' | sort -u | awk '!/ /'
--- a/packer/build.pkr.hcl
+++ b/packer/build.pkr.hcl
@ -28,7 +28,6 @@ build {

    extra_arguments  = [
      "--extra-vars", "appliancetype=${source.name}",
-      "--extra-vars", "applianceversion=${var.appliance_version}",
      "--extra-vars", "ansible_ssh_pass=${var.ssh_password}",
      "--extra-vars", "docker_username=${var.docker_username}",
      "--extra-vars", "docker_password=${var.docker_password}",
@ -46,7 +45,7 @@ build {
      " -ManifestFileName '/scratch/bld_${var.vm_name}_${source.name}.mf'",
      "ovftool --acceptAllEulas --allowExtraConfig --overwrite \\",
      " '/scratch/bld_${var.vm_name}_${source.name}.ovf' \\",
-      " /output/airgapped-k8s-${var.appliance_version}+${var.k8s_version}-${source.name}.ova"
+      " /output/airgapped-k8s-${var.k8s_version}.${source.name}.ova"
    ]
  }
 }
--- a/packer/variables.pkr.hcl
+++ b/packer/variables.pkr.hcl
@ -34,5 +34,4 @@ variable "docker_password" {
    sensitive = true
 }

-variable "appliance_version" {}
 variable "k8s_version" {}
Author	SHA1	Message	Date
Danny Bessems	8fba1c410e	fix: Incorrect indentation causing malformed PEM file Some checks failed continuous-integration/drone/push Build is failing Details	2023-07-07 10:29:57 +02:00
Danny Bessems	e2e009aa3c	fix: Refactor kustomize templates All checks were successful continuous-integration/drone/push Build is passing Details	2023-07-06 13:00:08 +02:00
Danny Bessems	ef81cbfe6a	feat:Explicitly configure NTP servers (WIP) All checks were successful continuous-integration/drone/push Build is passing Details	2023-06-03 10:07:09 +02:00
Danny Bessems	43095c5a1e	feat:Upgrade components	2023-06-03 10:06:33 +02:00