Bootstrap HA k3s server on 3 Harvester VM nodes

manifests/cloud-config-secret.yaml

@@ -0,0 +1,36 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: kairos-cloud-config
+  namespace: default
+type: Opaque
+stringData:
+  userdata: |
+    #cloud-config
+    hostname: "kairos-{{ trunc 5 .MachineID }}"
+    install:
+      auto: true
+      device: "auto"
+      reboot: true
+    bundles:
+      - targets:
+        - "run://quay.io/kairos/community-bundles:qemu-guest-agent-main"
+    users:
+      - name: kairos
+        passwd: kairos
+        groups: ["admin"]
+        ssh_authorized_keys:
+          - "ssh-rsa AAAAB3NzaC1yc2EAAAABJQAAAQEAiRc7Og+cRJGFwdUzgpX9YqvVenTk54N4kqM7emEfYHdsJLMjKQyxr8hklHmsam5dzxx3itFzc6SLf/ldJJ2JZuzE5FiCqUXXv4UFwN6HF5xqn7PTLicvWZH93H4m1gOlD5Dfzi4Es34v5zRBwbMScOgekk/LweTgl35jGKDgMP5DjGTqkPf7Ndh9+iuQrz99JEr8egl3bj+jIlKjScfaQbbnu3AJIRwZwTKgw0AOkLliQdEPNLvG5/ZImxJG4oHV9/uNkfdJObLjT1plR1HbVNskV5fuRNE/vnUiWl9jAJ1RT83GOqV0sQ+Q7p214fkgqb3JPvci/s0Bb7RA85hBEQ== djpbessems"
+    kubevip:
+      eip: "192.168.154.249"
+    p2p:
+      disable_dht: true
+      vpn:
+        create: false
+        use: false
+      network_token: b3RwOgogIGRodDoKICAgIGludGVydmFsOiA5MjIzMzcyMDM2ODU0Nzc1ODA3CiAgICBrZXk6IGNCV2hNcWFvdDltejFIbHN2bVpJZjRnMnJ1bmZZY1g4c1RlSWVKZDhPZzQKICAgIGxlbmd0aDogNDMKICBjcnlwdG86CiAgICBpbnRlcnZhbDogOTIyMzM3MjAzNjg1NDc3NTgwNwogICAga2V5OiBSR2hyanlpekVqcWJ5aWNnRU1zTXFpM0xpcjlWSllzZnd4cXRtUU9hdEl3CiAgICBsZW5ndGg6IDQzCnJvb206IGpNNVJJeXFKd1RqWnBrcDN0a1hUalRjQ3Z1Y2JzckFTckoxSVQ2OVpHSjEKcmVuZGV6dm91czogdTRkUmZrTGFiczZTMlpLQUJ1ZDJiV0pnb0VVcnlIbVNEVjNqbEw5S0pieAptZG5zOiBVeDFGZ2pOYVNXZHVEV3NTeG50TlJhRHVhc0pMMmZlNXhpb2ZQNG9vUTNMCm1heF9tZXNzYWdlX3NpemU6IDIwOTcxNTIwCg==
+      auto:
+        enable: true
+        ha:
+          enable: true
+          master_nodes: 2 # Total 3 masters (1 initial + 2 extra)

manifests/harvester-vms.yaml

@@ -0,0 +1,256 @@
+apiVersion: kubevirt.io/v1
+kind: VirtualMachine
+metadata:
+  name: kairos-node-1
+  namespace: default
+  annotations:
+    harvesterhci.io/volumeClaimTemplates: |-
+      [
+        {
+          "metadata": {
+            "name": "kairos-node-1-disk-0",
+            "annotations": {
+              "harvesterhci.io/imageId": "default/image-s9dln",
+              "harvesterhci.io/delete-after-vm-termination": "true"
+            }
+          },
+          "spec": {
+            "accessModes": ["ReadWriteMany"],
+            "resources": {"requests": {"storage": "1Gi"}},
+            "volumeMode": "Block",
+            "storageClassName": "longhorn-image-s9dln"
+          }
+        },
+        {
+          "metadata": {
+            "name": "kairos-node-1-disk-1",
+            "annotations": {
+              "harvesterhci.io/delete-after-vm-termination": "true"
+            }
+          },
+          "spec": {
+            "accessModes": ["ReadWriteMany"],
+            "resources": {"requests": {"storage": "50Gi"}},
+            "volumeMode": "Block",
+            "storageClassName": "harvester-longhorn"
+          }
+        }
+      ]
+    harvesterhci.io/vmRunStrategy: RerunOnFailure
+  labels:
+    harvesterhci.io/vmName: kairos-node-1
+spec:
+  runStrategy: RerunOnFailure
+  template:
+    metadata:
+      labels:
+        harvesterhci.io/vmName: kairos-node-1
+    spec:
+      domain:
+        cpu: {cores: 2}
+        resources:
+          limits:
+            memory: 4Gi
+          requests:
+            memory: 4Gi
+        machine: {type: q35}
+        features:
+          acpi: {enabled: true}
+        devices:
+          inputs:
+            - bus: usb
+              name: tablet
+              type: tablet
+          interfaces:
+            - model: virtio
+              name: default
+              bridge: {}
+          disks:
+            - name: disk-0
+              cdrom: {bus: sata}
+              bootOrder: 2
+            - name: disk-1
+              disk: {bus: virtio}
+              bootOrder: 1
+            - name: cloudinitdisk
+              disk: {bus: virtio}
+      networks:
+        - name: default
+          multus:
+            networkName: default/vmn-lan
+      volumes:
+        - name: disk-0
+          persistentVolumeClaim:
+            claimName: kairos-node-1-disk-0
+        - name: disk-1
+          persistentVolumeClaim:
+            claimName: kairos-node-1-disk-1
+        - name: cloudinitdisk
+          cloudInitNoCloud:
+            secretRef: {name: kairos-cloud-config}
+---
+apiVersion: kubevirt.io/v1
+kind: VirtualMachine
+metadata:
+  name: kairos-node-2
+  namespace: default
+  annotations:
+    harvesterhci.io/volumeClaimTemplates: |-
+      [
+        {
+          "metadata": {
+            "name": "kairos-node-2-disk-0",
+            "annotations": {"harvesterhci.io/imageId": "default/image-s9dln"}
+          },
+          "spec": {
+            "accessModes": ["ReadWriteMany"],
+            "resources": {"requests": {"storage": "1Gi"}},
+            "volumeMode": "Block",
+            "storageClassName": "longhorn-image-s9dln"
+          }
+        },
+        {
+          "metadata": {"name": "kairos-node-2-disk-1"},
+          "spec": {
+            "accessModes": ["ReadWriteMany"],
+            "resources": {"requests": {"storage": "50Gi"}},
+            "volumeMode": "Block",
+            "storageClassName": "harvester-longhorn"
+          }
+        }
+      ]
+    harvesterhci.io/vmRunStrategy: RerunOnFailure
+  labels:
+    harvesterhci.io/vmName: kairos-node-2
+spec:
+  runStrategy: RerunOnFailure
+  template:
+    metadata:
+      labels:
+        harvesterhci.io/vmName: kairos-node-2
+    spec:
+      domain:
+        cpu: {cores: 2}
+        resources:
+          limits:
+            memory: 4Gi
+          requests:
+            memory: 4Gi
+        machine: {type: q35}
+        features:
+          acpi: {enabled: true}
+        devices:
+          inputs:
+            - bus: usb
+              name: tablet
+              type: tablet
+          interfaces:
+            - model: virtio
+              name: default
+              bridge: {}
+          disks:
+            - name: disk-0
+              cdrom: {bus: sata}
+              bootOrder: 2
+            - name: disk-1
+              disk: {bus: virtio}
+              bootOrder: 1
+            - name: cloudinitdisk
+              disk: {bus: virtio}
+      networks:
+        - name: default
+          multus:
+            networkName: default/vmn-lan
+      volumes:
+        - name: disk-0
+          persistentVolumeClaim:
+            claimName: kairos-node-2-disk-0
+        - name: disk-1
+          persistentVolumeClaim:
+            claimName: kairos-node-2-disk-1
+        - name: cloudinitdisk
+          cloudInitNoCloud:
+            secretRef: {name: kairos-cloud-config}
+---
+apiVersion: kubevirt.io/v1
+kind: VirtualMachine
+metadata:
+  name: kairos-node-3
+  namespace: default
+  annotations:
+    harvesterhci.io/volumeClaimTemplates: |-
+      [
+        {
+          "metadata": {
+            "name": "kairos-node-3-disk-0",
+            "annotations": {"harvesterhci.io/imageId": "default/image-s9dln"}
+          },
+          "spec": {
+            "accessModes": ["ReadWriteMany"],
+            "resources": {"requests": {"storage": "1Gi"}},
+            "volumeMode": "Block",
+            "storageClassName": "longhorn-image-s9dln"
+          }
+        },
+        {
+          "metadata": {"name": "kairos-node-3-disk-1"},
+          "spec": {
+            "accessModes": ["ReadWriteMany"],
+            "resources": {"requests": {"storage": "50Gi"}},
+            "volumeMode": "Block",
+            "storageClassName": "harvester-longhorn"
+          }
+        }
+      ]
+    harvesterhci.io/vmRunStrategy: RerunOnFailure
+  labels:
+    harvesterhci.io/vmName: kairos-node-3
+spec:
+  runStrategy: RerunOnFailure
+  template:
+    metadata:
+      labels:
+        harvesterhci.io/vmName: kairos-node-3
+    spec:
+      domain:
+        cpu: {cores: 2}
+        resources:
+          limits:
+            memory: 4Gi
+          requests:
+            memory: 4Gi
+        machine: {type: q35}
+        features:
+          acpi: {enabled: true}
+        devices:
+          inputs:
+            - bus: usb
+              name: tablet
+              type: tablet
+          interfaces:
+            - model: virtio
+              name: default
+              bridge: {}
+          disks:
+            - name: disk-0
+              cdrom: {bus: sata}
+              bootOrder: 2
+            - name: disk-1
+              disk: {bus: virtio}
+              bootOrder: 1
+            - name: cloudinitdisk
+              disk: {bus: virtio}
+      networks:
+        - name: default
+          multus:
+            networkName: default/vmn-lan
+      volumes:
+        - name: disk-0
+          persistentVolumeClaim:
+            claimName: kairos-node-3-disk-0
+        - name: disk-1
+          persistentVolumeClaim:
+            claimName: kairos-node-3-disk-1
+        - name: cloudinitdisk
+          cloudInitNoCloud:
+            secretRef: {name: kairos-cloud-config}

manifests/harvester-vms.yaml.OLD

@@ -0,0 +1,225 @@
+apiVersion: kubevirt.io/v1
+kind: VirtualMachine
+metadata:
+  name: kairos-node-1
+  namespace: default
+spec:
+  running: true
+  template:
+    spec:
+      domain:
+        cpu:
+          cores: 2
+        resources:
+          requests:
+            memory: 4Gi
+          limits:
+            memory: 4Gi
+        devices:
+          disks:
+            - bootOrder: 1
+              cdrom:
+                bus: sata
+              name: cdrom-iso
+            - bootOrder: 2
+              disk:
+                bus: virtio
+              name: persistent-disk
+            - disk:
+                bus: virtio
+              name: cloudinitdisk
+      volumes:
+        - name: cdrom-iso
+          dataVolume:
+            name: kairos-iso-vol-1
+        - name: persistent-disk
+          dataVolume:
+            name: kairos-data-vol-1
+        - name: cloudinitdisk
+          cloudInitNoCloud:
+            secretRef:
+              name: kairos-cloud-config
+---
+apiVersion: cdi.kubevirt.io/v1beta1
+kind: DataVolume
+metadata:
+  name: kairos-iso-vol-1
+  namespace: default
+  annotations:
+    harvesterhci.io/imageId: "default/image-s9dln"
+spec:
+  source:
+    blank: {}
+  storage:
+    accessModes:
+      - ReadWriteMany
+    resources:
+      requests:
+        storage: 1Gi
+    storageClassName: harvester-longhorn
+    volumeMode: Block
+---
+apiVersion: cdi.kubevirt.io/v1beta1
+kind: DataVolume
+metadata:
+  name: kairos-data-vol-1
+  namespace: default
+spec:
+  source:
+    blank: {}
+  storage:
+    resources:
+      requests:
+        storage: 50Gi
+    storageClassName: harvester-longhorn
+---
+apiVersion: kubevirt.io/v1
+kind: VirtualMachine
+metadata:
+  name: kairos-node-2
+  namespace: default
+spec:
+  running: true
+  template:
+    spec:
+      domain:
+        cpu:
+          cores: 2
+        resources:
+          requests:
+            memory: 4Gi
+          limits:
+            memory: 4Gi
+        devices:
+          disks:
+            - bootOrder: 1
+              cdrom:
+                bus: sata
+              name: cdrom-iso
+            - bootOrder: 2
+              disk:
+                bus: virtio
+              name: persistent-disk
+            - disk:
+                bus: virtio
+              name: cloudinitdisk
+      volumes:
+        - name: cdrom-iso
+          dataVolume:
+            name: kairos-iso-vol-2
+        - name: persistent-disk
+          dataVolume:
+            name: kairos-data-vol-2
+        - name: cloudinitdisk
+          cloudInitNoCloud:
+            secretRef:
+              name: kairos-cloud-config
+---
+apiVersion: cdi.kubevirt.io/v1beta1
+kind: DataVolume
+metadata:
+  name: kairos-iso-vol-2
+  namespace: default
+  annotations:
+    harvesterhci.io/imageId: "default/image-s9dln"
+spec:
+  source:
+    blank: {}
+  storage:
+    accessModes:
+      - ReadWriteMany
+    resources:
+      requests:
+        storage: 1Gi
+    storageClassName: harvester-longhorn
+    volumeMode: Block
+---
+apiVersion: cdi.kubevirt.io/v1beta1
+kind: DataVolume
+metadata:
+  name: kairos-data-vol-2
+  namespace: default
+spec:
+  source:
+    blank: {}
+  storage:
+    resources:
+      requests:
+        storage: 50Gi
+    storageClassName: harvester-longhorn
+---
+apiVersion: kubevirt.io/v1
+kind: VirtualMachine
+metadata:
+  name: kairos-node-3
+  namespace: default
+spec:
+  running: true
+  template:
+    spec:
+      domain:
+        cpu:
+          cores: 2
+        resources:
+          requests:
+            memory: 4Gi
+          limits:
+            memory: 4Gi
+        devices:
+          disks:
+            - bootOrder: 1
+              cdrom:
+                bus: sata
+              name: cdrom-iso
+            - bootOrder: 2
+              disk:
+                bus: virtio
+              name: persistent-disk
+            - disk:
+                bus: virtio
+              name: cloudinitdisk
+      volumes:
+        - name: cdrom-iso
+          dataVolume:
+            name: kairos-iso-vol-3
+        - name: persistent-disk
+          dataVolume:
+            name: kairos-data-vol-3
+        - name: cloudinitdisk
+          cloudInitNoCloud:
+            secretRef:
+              name: kairos-cloud-config
+---
+apiVersion: cdi.kubevirt.io/v1beta1
+kind: DataVolume
+metadata:
+  name: kairos-iso-vol-3
+  namespace: default
+  annotations:
+    harvesterhci.io/imageId: "default/image-s9dln"
+spec:
+  source:
+    blank: {}
+  storage:
+    accessModes:
+      - ReadWriteMany
+    resources:
+      requests:
+        storage: 1Gi
+    storageClassName: harvester-longhorn
+    volumeMode: Block
+---
+apiVersion: cdi.kubevirt.io/v1beta1
+kind: DataVolume
+metadata:
+  name: kairos-data-vol-3
+  namespace: default
+spec:
+  source:
+    blank: {}
+  storage:
+    resources:
+      requests:
+        storage: 50Gi
+    storageClassName: harvester-longhorn
+---

zarf.yaml

@@ -0,0 +1,15 @@
+kind: ZarfPackageConfig
+metadata:
+  name: harvester-bootstrap-rancher
+#  version: 26.0.0
+#  description: |
+#    "Deploys a HA k3s cluster on top of a Harvester cluster, installs Rancher and imports the Harvester local cluster"
+
+components:
+  - name: kairos-cluster
+    required: true
+    manifests:
+      - name: cluster-specs
+        files:
+          - manifests/cloud-config-secret.yaml
+          - manifests/harvester-vms.yaml