djpbessems/Go.Rig-Operator
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Drop initial code

Browse Source
This commit is contained in:
Danny Bessems
2026-01-15 09:58:01 +00:00
parent 227d957219
commit 1e7c9ba5cb
228 changed files with 19883 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

157
deploy/k8s-provisioner/internal/controller/cluster_controller.go Normal file
View File

@@ -0,0 +1,157 @@
package controller
import (
"context"
_ "embed"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
"sigs.k8s.io/controller-runtime/pkg/log"
k8sprovisionerv1alpha1 "vanderlande.com/appstack/k8s-provisioner/api/v1alpha1"
"vanderlande.com/appstack/k8s-provisioner/internal/harvester"
"vanderlande.com/appstack/k8s-provisioner/internal/helm"
"vanderlande.com/appstack/k8s-provisioner/internal/templates"
"vanderlande.com/appstack/k8s-provisioner/internal/values"
)
type ClusterReconciler struct {
client.Client
Scheme *runtime.Scheme
}
// Internal Struct for mapping NodePools to Helm Values
type HelmNodePool struct {
Name string `json:"name"`
DisplayName string `json:"displayName"`
Quantity int `json:"quantity"`
Etcd bool `json:"etcd"`
ControlPlane bool `json:"controlplane"`
Worker bool `json:"worker"`
Paused bool `json:"paused"`
CpuCount int `json:"cpuCount"`
DiskSize int `json:"diskSize"`
ImageName string `json:"imageName"`
MemorySize int `json:"memorySize"`
NetworkName string `json:"networkName"`
SshUser string `json:"sshUser"`
VmNamespace string `json:"vmNamespace"`
UserData string `json:"userData"`
}
// +kubebuilder:rbac:groups=k8sprovisioner.appstack.io,resources=clusters,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=k8sprovisioner.appstack.io,resources=clusters/status,verbs=get;update;patch
// +kubebuilder:rbac:groups=k8sprovisioner.appstack.io,resources=infras,verbs=get;list;watch
// +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;create;update;patch;delete
const clusterFinalizer = "k8sprovisioner.appstack.io/finalizer"
func (r *ClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
l := log.FromContext(ctx)
// Initialize Managers
hvManager := harvester.NewIdentityManager(r.Client, r.Scheme)
// 1. Fetch Cluster
var cluster k8sprovisionerv1alpha1.Cluster
if err := r.Get(ctx, req.NamespacedName, &cluster); err != nil {
return ctrl.Result{}, client.IgnoreNotFound(err)
}
// 2. Handle Deletion
if !cluster.ObjectMeta.DeletionTimestamp.IsZero() {
if controllerutil.ContainsFinalizer(&cluster, clusterFinalizer) {
l.Info("Processing Cluster Deletion...")
// A. Uninstall Helm
helmCfg := helm.Config{Namespace: req.Namespace, ReleaseName: req.Name}
if err := helm.Uninstall(helmCfg); err != nil {
return ctrl.Result{}, err
}
// B. Cleanup Harvester (Using Manager)
hvManager.Cleanup(ctx, &cluster)
// C. Remove Finalizer
controllerutil.RemoveFinalizer(&cluster, clusterFinalizer)
if err := r.Update(ctx, &cluster); err != nil {
return ctrl.Result{}, err
}
}
return ctrl.Result{}, nil
}
// 3. Add Finalizer
if !controllerutil.ContainsFinalizer(&cluster, clusterFinalizer) {
controllerutil.AddFinalizer(&cluster, clusterFinalizer)
if err := r.Update(ctx, &cluster); err != nil {
return ctrl.Result{}, err
}
}
// 4. Fetch Infra
var infra k8sprovisionerv1alpha1.Infra
if err := r.Get(ctx, types.NamespacedName{Name: cluster.Spec.InfraRef, Namespace: req.Namespace}, &infra); err != nil {
return ctrl.Result{}, err
}
// =========================================================
// 5. SECURE HARVESTER IDENTITY (Simplified)
// =========================================================
// The manager handles looking up Rancher creds, minting tokens,
// saving secrets, and updating the Cluster status.
generatedSecretName, err := hvManager.Ensure(ctx, &cluster, &infra)
if err != nil {
return ctrl.Result{}, err
}
// =========================================================
// 6. HELM VALUES GENERATION
// =========================================================
vb := values.NewBuilder(
&cluster,
&infra,
templates.BaseValuesYAML,
generatedSecretName,
req.Namespace,
)
helmValues, err := vb.Build()
if err != nil {
l.Error(err, "Failed to generate helm values")
return ctrl.Result{}, err
}
chartSpec := vb.GetChartConfig()
// 7. Trigger Helm Apply
l.Info("Syncing Helm Release", "Release", req.Name)
helmCfg := helm.Config{
Namespace: req.Namespace,
ReleaseName: req.Name,
RepoURL: chartSpec.Repo,
ChartName: chartSpec.Name,
Version: chartSpec.Version,
Values: helmValues,
}
if err := helm.Apply(helmCfg); err != nil {
l.Error(err, "Helm Apply Failed")
return ctrl.Result{}, err
}
return ctrl.Result{}, nil
}
func (r *ClusterReconciler) SetupWithManager(mgr ctrl.Manager) error {
return ctrl.NewControllerManagedBy(mgr).
For(&k8sprovisionerv1alpha1.Cluster{}).
Complete(r)
}

84
deploy/k8s-provisioner/internal/controller/cluster_controller_test.go Normal file
View File

@@ -0,0 +1,84 @@
/*
Copyright 2026.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package controller
import (
"context"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
k8sprovisionerv1alpha1 "vanderlande.com/appstack/k8s-provisioner/api/v1alpha1"
)
var _ = Describe("Cluster Controller", func() {
Context("When reconciling a resource", func() {
const resourceName = "test-resource"
ctx := context.Background()
typeNamespacedName := types.NamespacedName{
Name: resourceName,
Namespace: "default", // TODO(user):Modify as needed
}
cluster := &k8sprovisionerv1alpha1.Cluster{}
BeforeEach(func() {
By("creating the custom resource for the Kind Cluster")
err := k8sClient.Get(ctx, typeNamespacedName, cluster)
if err != nil && errors.IsNotFound(err) {
resource := &k8sprovisionerv1alpha1.Cluster{
ObjectMeta: metav1.ObjectMeta{
Name: resourceName,
Namespace: "default",
},
// TODO(user): Specify other spec details if needed.
}
Expect(k8sClient.Create(ctx, resource)).To(Succeed())
}
})
AfterEach(func() {
// TODO(user): Cleanup logic after each test, like removing the resource instance.
resource := &k8sprovisionerv1alpha1.Cluster{}
err := k8sClient.Get(ctx, typeNamespacedName, resource)
Expect(err).NotTo(HaveOccurred())
By("Cleanup the specific resource instance Cluster")
Expect(k8sClient.Delete(ctx, resource)).To(Succeed())
})
It("should successfully reconcile the resource", func() {
By("Reconciling the created resource")
controllerReconciler := &ClusterReconciler{
Client: k8sClient,
Scheme: k8sClient.Scheme(),
}
_, err := controllerReconciler.Reconcile(ctx, reconcile.Request{
NamespacedName: typeNamespacedName,
})
Expect(err).NotTo(HaveOccurred())
// TODO(user): Add more specific assertions depending on your controller's reconciliation logic.
// Example: If you expect a certain status condition after reconciliation, verify it here.
})
})
})

116
deploy/k8s-provisioner/internal/controller/suite_test.go Normal file
View File

@@ -0,0 +1,116 @@
/*
Copyright 2026.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package controller
import (
"context"
"os"
"path/filepath"
"testing"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/rest"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/envtest"
logf "sigs.k8s.io/controller-runtime/pkg/log"
"sigs.k8s.io/controller-runtime/pkg/log/zap"
k8sprovisionerv1alpha1 "vanderlande.com/appstack/k8s-provisioner/api/v1alpha1"
// +kubebuilder:scaffold:imports
)
// These tests use Ginkgo (BDD-style Go testing framework). Refer to
// http://onsi.github.io/ginkgo/ to learn more about Ginkgo.
var (
ctx context.Context
cancel context.CancelFunc
testEnv *envtest.Environment
cfg *rest.Config
k8sClient client.Client
)
func TestControllers(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Controller Suite")
}
var _ = BeforeSuite(func() {
logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true)))
ctx, cancel = context.WithCancel(context.TODO())
var err error
err = k8sprovisionerv1alpha1.AddToScheme(scheme.Scheme)
Expect(err).NotTo(HaveOccurred())
// +kubebuilder:scaffold:scheme
By("bootstrapping test environment")
testEnv = &envtest.Environment{
CRDDirectoryPaths: []string{filepath.Join("..", "..", "config", "crd", "bases")},
ErrorIfCRDPathMissing: true,
}
// Retrieve the first found binary directory to allow running tests from IDEs
if getFirstFoundEnvTestBinaryDir() != "" {
testEnv.BinaryAssetsDirectory = getFirstFoundEnvTestBinaryDir()
}
// cfg is defined in this file globally.
cfg, err = testEnv.Start()
Expect(err).NotTo(HaveOccurred())
Expect(cfg).NotTo(BeNil())
k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme})
Expect(err).NotTo(HaveOccurred())
Expect(k8sClient).NotTo(BeNil())
})
var _ = AfterSuite(func() {
By("tearing down the test environment")
cancel()
err := testEnv.Stop()
Expect(err).NotTo(HaveOccurred())
})
// getFirstFoundEnvTestBinaryDir locates the first binary in the specified path.
// ENVTEST-based tests depend on specific binaries, usually located in paths set by
// controller-runtime. When running tests directly (e.g., via an IDE) without using
// Makefile targets, the 'BinaryAssetsDirectory' must be explicitly configured.
//
// This function streamlines the process by finding the required binaries, similar to
// setting the 'KUBEBUILDER_ASSETS' environment variable. To ensure the binaries are
// properly set up, run 'make setup-envtest' beforehand.
func getFirstFoundEnvTestBinaryDir() string {
basePath := filepath.Join("..", "..", "bin", "k8s")
entries, err := os.ReadDir(basePath)
if err != nil {
logf.Log.Error(err, "Failed to read directory", "path", basePath)
return ""
}
for _, entry := range entries {
if entry.IsDir() {
return filepath.Join(basePath, entry.Name())
}
}
return ""
}

229
deploy/k8s-provisioner/internal/harvester/factory.go Normal file
View File

@@ -0,0 +1,229 @@
package harvester
import (
"context"
"encoding/base64"
"fmt"
"time"
authenticationv1 "k8s.io/api/authentication/v1"
corev1 "k8s.io/api/core/v1"
rbacv1 "k8s.io/api/rbac/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/clientcmd"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/log"
k8sprovisionerv1alpha1 "vanderlande.com/appstack/k8s-provisioner/api/v1alpha1"
)
// TryCleanup performs a "Best Effort" cleanup of Harvester resources.
func TryCleanup(ctx context.Context, k8sClient client.Client, infraRefName, namespace, saName string) {
l := log.FromContext(ctx)
// 1. Fetch Infra
var infra k8sprovisionerv1alpha1.Infra
if err := k8sClient.Get(ctx, types.NamespacedName{Name: infraRefName, Namespace: namespace}, &infra); err != nil {
l.Info("Cleanup skipped: Infra object not found")
return
}
vmNamespace := infra.Spec.VmNamespace
if vmNamespace == "" {
vmNamespace = "default"
}
// 2. Fetch Master Credential
rancherCredName := infra.Spec.CloudCredentialSecret
var rancherSecret corev1.Secret
if err := k8sClient.Get(ctx, types.NamespacedName{Name: rancherCredName, Namespace: "cattle-global-data"}, &rancherSecret); err != nil {
l.Info("Cleanup skipped: Master Credential Secret not found")
return
}
// 3. Extract Kubeconfig
var kubeBytes []byte
if len(rancherSecret.Data["harvestercredentialConfig-kubeconfigContent"]) > 0 {
kubeBytes = rancherSecret.Data["harvestercredentialConfig-kubeconfigContent"]
} else if len(rancherSecret.Data["credential"]) > 0 {
kubeBytes = rancherSecret.Data["credential"]
} else {
return
}
// 4. Cleanup
if err := deleteHarvesterResources(ctx, kubeBytes, saName, vmNamespace); err != nil {
l.Error(err, "Failed to cleanup Harvester resources (ignoring)")
} else {
l.Info("Harvester resources deleted successfully")
}
}
// Internal helper for cleanup
func deleteHarvesterResources(ctx context.Context, masterKubeconfig []byte, serviceAccountName, vmNamespace string) error {
restConfig, err := clientcmd.RESTConfigFromKubeConfig(masterKubeconfig)
if err != nil {
return err
}
hvClient, err := kubernetes.NewForConfig(restConfig)
if err != nil {
return err
}
deletePolicy := metav1.DeletePropagationBackground
deleteOpts := metav1.DeleteOptions{PropagationPolicy: &deletePolicy}
// 1. Delete Global CSI Binding (ClusterRoleBinding)
csiBindingName := fmt.Sprintf("%s-csi-binding", serviceAccountName)
err = hvClient.RbacV1().ClusterRoleBindings().Delete(ctx, csiBindingName, deleteOpts)
if err != nil && !apierrors.IsNotFound(err) {
return err
}
// 2. Delete Cloud Provider Binding (RoleBinding in VM Namespace)
cpBindingName := fmt.Sprintf("%s-cloud-binding", serviceAccountName)
err = hvClient.RbacV1().RoleBindings(vmNamespace).Delete(ctx, cpBindingName, deleteOpts)
if err != nil && !apierrors.IsNotFound(err) {
return err
}
// 3. Delete ServiceAccount (VM Namespace)
err = hvClient.CoreV1().ServiceAccounts(vmNamespace).Delete(ctx, serviceAccountName, deleteOpts)
if err != nil && !apierrors.IsNotFound(err) {
return err
}
return nil
}
// EnsureCredential mints a dedicated ServiceAccount in the specific VM Namespace
func EnsureCredential(ctx context.Context, masterKubeconfig []byte, clusterName, targetNamespace, vmNamespace, harvesterURL string) (*corev1.Secret, string, time.Time, error) {
// --- PHASE 1: Connect (Proxy/Master Config) ---
restConfig, err := clientcmd.RESTConfigFromKubeConfig(masterKubeconfig)
if err != nil {
return nil, "", time.Time{}, fmt.Errorf("invalid rancher cloud credential kubeconfig: %w", err)
}
hvClient, err := kubernetes.NewForConfig(restConfig)
if err != nil {
return nil, "", time.Time{}, err
}
// --- PHASE 2: Create Identity (SA & Bindings) ---
if vmNamespace == "" {
vmNamespace = "default"
}
saName := fmt.Sprintf("prov-%s", clusterName)
// A. Create ServiceAccount
sa := &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: saName, Namespace: vmNamespace}}
if _, err := hvClient.CoreV1().ServiceAccounts(vmNamespace).Create(ctx, sa, metav1.CreateOptions{}); err != nil {
if !apierrors.IsAlreadyExists(err) {
return nil, "", time.Time{}, err
}
}
// B. Create RoleBinding (VM Namespace)
rb := &rbacv1.RoleBinding{
ObjectMeta: metav1.ObjectMeta{Name: saName + "-cloud-binding", Namespace: vmNamespace},
Subjects: []rbacv1.Subject{{Kind: "ServiceAccount", Name: saName, Namespace: vmNamespace}},
RoleRef: rbacv1.RoleRef{Kind: "ClusterRole", Name: "harvesterhci.io:cloudprovider", APIGroup: "rbac.authorization.k8s.io"},
}
if _, err := hvClient.RbacV1().RoleBindings(vmNamespace).Create(ctx, rb, metav1.CreateOptions{}); err != nil {
if !apierrors.IsAlreadyExists(err) { /* Ignore */
}
}
// C. Create ClusterRoleBinding (Global)
crb := &rbacv1.ClusterRoleBinding{
ObjectMeta: metav1.ObjectMeta{Name: saName + "-csi-binding"},
Subjects: []rbacv1.Subject{{Kind: "ServiceAccount", Name: saName, Namespace: vmNamespace}},
RoleRef: rbacv1.RoleRef{Kind: "ClusterRole", Name: "harvesterhci.io:csi-driver", APIGroup: "rbac.authorization.k8s.io"},
}
if _, err := hvClient.RbacV1().ClusterRoleBindings().Create(ctx, crb, metav1.CreateOptions{}); err != nil {
if !apierrors.IsAlreadyExists(err) { /* Ignore */
}
}
// D. Mint Token
ttlSeconds := int64(315360000)
tokenRequest, err := hvClient.CoreV1().ServiceAccounts(vmNamespace).CreateToken(ctx, saName, &authenticationv1.TokenRequest{
Spec: authenticationv1.TokenRequestSpec{ExpirationSeconds: &ttlSeconds},
}, metav1.CreateOptions{})
if err != nil {
return nil, "", time.Time{}, fmt.Errorf("failed to mint harvester token: %w", err)
}
expiryTime := time.Now().Add(time.Duration(ttlSeconds) * time.Second)
// --- PHASE 3: Determine URL & CA ---
// 1. URL: Use the explicitly provided HarvesterURL
if harvesterURL == "" {
// Fallback to Proxy if user forgot to set it (Safety net)
harvesterURL = restConfig.Host
}
// 2. CA: Fetch the internal Harvester CA
// (Required because the proxy CA won't match the direct IP/URL)
harvesterCA := restConfig.CAData
caConfigMap, err := hvClient.CoreV1().ConfigMaps("default").Get(ctx, "kube-root-ca.crt", metav1.GetOptions{})
if err == nil {
if caStr, ok := caConfigMap.Data["ca.crt"]; ok {
harvesterCA = []byte(caStr)
}
}
// --- PHASE 4: Construct Kubeconfig ---
caData := base64.StdEncoding.EncodeToString(harvesterCA)
token := tokenRequest.Status.Token
// Ensure "namespace" aligns vertically with "cluster" and "user"
newKubeconfig := fmt.Sprintf(
`apiVersion: v1
kind: Config
clusters:
- name: harvester
cluster:
server: %s
certificate-authority-data: %s
users:
- name: provisioner
user:
token: %s
contexts:
- name: default
context:
cluster: harvester
user: provisioner
namespace: %s
current-context: default
`, harvesterURL, caData, token, vmNamespace)
// --- PHASE 5: Create Secret ---
secretName := fmt.Sprintf("harvesterconfig-%s", clusterName)
secret := &corev1.Secret{
TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"},
ObjectMeta: metav1.ObjectMeta{
Name: secretName,
Namespace: targetNamespace,
Annotations: map[string]string{
"v2prov-secret-authorized-for-cluster": clusterName,
"v2prov-authorized-secret-deletes-on-cluster-removal": "true",
},
Labels: map[string]string{
"cattle.io/creator": "k8s-provisioner",
},
},
Type: "Opaque",
StringData: map[string]string{
"credential": newKubeconfig,
},
}
return secret, saName, expiryTime, nil
}

126
deploy/k8s-provisioner/internal/helm/client.go Normal file
View File

@@ -0,0 +1,126 @@
package helm
import (
"fmt"
"log"
"os"
"helm.sh/helm/v3/pkg/action"
"helm.sh/helm/v3/pkg/chart/loader"
"helm.sh/helm/v3/pkg/cli"
"helm.sh/helm/v3/pkg/registry" // [NEW] Required for OCI
"helm.sh/helm/v3/pkg/storage/driver"
"k8s.io/cli-runtime/pkg/genericclioptions"
)
type Config struct {
Namespace string
ReleaseName string
RepoURL string
ChartName string
Version string
Values map[string]interface{}
}
func Apply(cfg Config) error {
settings := cli.New()
// 1. Initialize Action Config
actionConfig := new(action.Configuration)
getter := genericclioptions.NewConfigFlags(false)
if err := actionConfig.Init(getter, cfg.Namespace, os.Getenv("HELM_DRIVER"), log.Printf); err != nil {
return fmt.Errorf("failed to init helm config: %w", err)
}
// 2. [NEW] Initialize OCI Registry Client
// This tells Helm how to talk to ghcr.io, docker.io, etc.
registryClient, err := registry.NewClient(
registry.ClientOptDebug(true),
registry.ClientOptEnableCache(true),
registry.ClientOptCredentialsFile(settings.RegistryConfig), // Uses ~/.config/helm/registry/config.json
)
if err != nil {
return fmt.Errorf("failed to init registry client: %w", err)
}
actionConfig.RegistryClient = registryClient
// 3. Setup Install Action
client := action.NewInstall(actionConfig)
client.Version = cfg.Version
client.Namespace = cfg.Namespace
client.ReleaseName = cfg.ReleaseName
client.CreateNamespace = true
if cfg.RepoURL != "" {
client.RepoURL = cfg.RepoURL
}
// 4. Locate Chart (Now supports oci:// because RegistryClient is set)
cp, err := client.ChartPathOptions.LocateChart(cfg.ChartName, settings)
if err != nil {
return fmt.Errorf("failed to locate chart %s: %w", cfg.ChartName, err)
}
chart, err := loader.Load(cp)
if err != nil {
return fmt.Errorf("failed to load chart: %w", err)
}
// 5. Install or Upgrade
histClient := action.NewHistory(actionConfig)
histClient.Max = 1
if _, err := histClient.Run(cfg.ReleaseName); err == driver.ErrReleaseNotFound {
fmt.Printf("Installing OCI Release %s...\n", cfg.ReleaseName)
_, err := client.Run(chart, cfg.Values)
return err
} else if err != nil {
return err
}
fmt.Printf("Upgrading OCI Release %s...\n", cfg.ReleaseName)
upgrade := action.NewUpgrade(actionConfig)
upgrade.Version = cfg.Version
upgrade.Namespace = cfg.Namespace
// Important: Upgrade also needs the RegistryClient, but it shares 'actionConfig'
// so it is already set up.
if cfg.RepoURL != "" {
upgrade.RepoURL = cfg.RepoURL
}
_, err = upgrade.Run(cfg.ReleaseName, chart, cfg.Values)
return err
}
func Uninstall(cfg Config) error {
settings := cli.New()
// 1. Initialize Action Config (Same as Apply)
actionConfig := new(action.Configuration)
getter := genericclioptions.NewConfigFlags(false)
if err := actionConfig.Init(getter, cfg.Namespace, os.Getenv("HELM_DRIVER"), log.Printf); err != nil {
return fmt.Errorf("failed to init helm config: %w", err)
}
// 2. Initialize OCI Registry Client (Crucial for OCI charts)
registryClient, err := registry.NewClient(
registry.ClientOptDebug(true),
registry.ClientOptEnableCache(true),
registry.ClientOptCredentialsFile(settings.RegistryConfig),
)
if err != nil {
return fmt.Errorf("failed to init registry client: %w", err)
}
actionConfig.RegistryClient = registryClient
// 3. Run Uninstall
client := action.NewUninstall(actionConfig)
// Don't fail if it's already gone
_, err = client.Run(cfg.ReleaseName)
if err != nil && err != driver.ErrReleaseNotFound {
return fmt.Errorf("failed to uninstall release: %w", err)
}
fmt.Printf("✅ Uninstalled Release %s\n", cfg.ReleaseName)
return nil
}

456
deploy/k8s-provisioner/internal/templates/base_values.yaml Normal file
View File

@@ -0,0 +1,456 @@
# ----------------------------------------------------------------
# BASE TEMPLATE (internal/templates/base_values.yaml)
# ----------------------------------------------------------------
_defaults:
helmChart:
repo: ""
name: "oci://ghcr.io/rancherfederal/charts/rancher-cluster-templates"
version: "0.7.2"
controlPlaneProfile:
cpuCores: 4
memoryGb: 8
diskGb: 40
userData: &userData |
#cloud-config
package_update: false
package_upgrade: false
snap:
commands:
00: snap refresh --hold=forever
package_reboot_if_required: true
packages:
- qemu-guest-agent
- yq
- jq
- curl
- wget
bootcmd:
- sysctl -w net.ipv6.conf.all.disable_ipv6=1
- sysctl -w net.ipv6.conf.default.disable_ipv6=1
write_files:
# ----------------------------------------------------------------
# 1. CNI Permission Fix Script & Cron (CIS 1.1.9 Persistence)
# ----------------------------------------------------------------
- path: /usr/local/bin/fix-cni-perms.sh
permissions: '0700'
owner: root:root
content: |
#!/bin/bash
# Wait 60s on boot for RKE2 to write files
[ "$1" == "boot" ] && sleep 60
# Enforce 600 on CNI files (CIS 1.1.9)
if [ -d /etc/cni/net.d ]; then
find /etc/cni/net.d -type f -exec chmod 600 {} \;
fi
if [ -d /var/lib/cni/networks ]; then
find /var/lib/cni/networks -type f -exec chmod 600 {} \;
fi
# Every RKE2 service restart can reset CNI file permissions, so we run
# this script on reboot and daily via cron to maintain CIS compliance.
- path: /etc/cron.d/cis-cni-fix
permissions: '0644'
owner: root:root
content: |
# Run on Reboot (with delay) to fix files created during startup
@reboot root /usr/local/bin/fix-cni-perms.sh boot
# Run once daily at 00:00 to correct any drift
0 0 * * * root /usr/local/bin/fix-cni-perms.sh
# ----------------------------------------------------------------
# 2. RKE2 Admission Config
# ----------------------------------------------------------------
- path: /etc/rancher/rke2/rke2-admission.yaml
permissions: '0600'
owner: root:root
content: |
apiVersion: apiserver.config.k8s.io/v1
kind: AdmissionConfiguration
plugins:
- name: PodSecurity
configuration:
apiVersion: pod-security.admission.config.k8s.io/v1beta1
kind: PodSecurityConfiguration
defaults:
enforce: "restricted"
enforce-version: "latest"
audit: "restricted"
audit-version: "latest"
warn: "restricted"
warn-version: "latest"
exemptions:
usernames: []
runtimeClasses: []
namespaces: [compliance-operator-system,kube-system, cis-operator-system, tigera-operator, calico-system, rke2-ingress-nginx, cattle-system, cattle-fleet-system, longhorn-system, cattle-neuvector-system]
- name: EventRateLimit
configuration:
apiVersion: eventratelimit.admission.k8s.io/v1alpha1
kind: Configuration
limits:
- type: Server
qps: 5000
burst: 20000
# ----------------------------------------------------------------
# 3. RKE2 Audit Policy
# ----------------------------------------------------------------
- path: /etc/rancher/rke2/audit-policy.yaml
permissions: '0600'
owner: root:root
content: |
apiVersion: audit.k8s.io/v1
kind: Policy
rules:
- level: None
users: ["system:kube-controller-manager", "system:kube-scheduler", "system:serviceaccount:kube-system:endpoint-controller"]
verbs: ["get", "update"]
resources:
- group: ""
resources: ["endpoints", "services", "services/status"]
- level: None
verbs: ["get"]
resources:
- group: ""
resources: ["nodes", "nodes/status", "pods", "pods/status"]
- level: None
users: ["kube-proxy"]
verbs: ["watch"]
resources:
- group: ""
resources: ["endpoints", "services", "services/status", "configmaps"]
- level: Metadata
resources:
- group: ""
resources: ["secrets", "configmaps"]
- level: RequestResponse
omitStages:
- RequestReceived
# ----------------------------------------------------------------
# 4. Static NetworkPolicies
# ----------------------------------------------------------------
- path: /var/lib/rancher/rke2/server/manifests/cis-network-policy.yaml
permissions: '0600'
owner: root:root
content: |
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: default-deny-ingress
namespace: default
spec:
podSelector: {}
policyTypes:
- Ingress
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: allow-all-metrics
namespace: kube-public
spec:
podSelector: {}
ingress:
- {}
policyTypes:
- Ingress
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: allow-all-system
namespace: kube-system
spec:
podSelector: {}
ingress:
- {}
policyTypes:
- Ingress
# ----------------------------------------------------------------
# 5. Service Account Hardening
# ----------------------------------------------------------------
- path: /var/lib/rancher/rke2/server/manifests/cis-sa-config.yaml
permissions: '0600'
owner: root:root
content: |
apiVersion: v1
kind: ServiceAccount
metadata:
name: default
namespace: default
automountServiceAccountToken: false
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: default
namespace: kube-system
automountServiceAccountToken: false
- path: /var/lib/rancher/rke2/server/manifests/cis-sa-cron.yaml
permissions: '0600'
owner: root:root
content: |
apiVersion: v1
kind: ServiceAccount
metadata: {name: sa-cleaner, namespace: kube-system}
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata: {name: sa-cleaner-role}
rules:
- apiGroups: [""]
resources: ["namespaces", "serviceaccounts"]
verbs: ["get", "list", "patch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata: {name: sa-cleaner-binding}
subjects: [{kind: ServiceAccount, name: sa-cleaner, namespace: kube-system}]
roleRef: {kind: ClusterRole, name: sa-cleaner-role, apiGroup: rbac.authorization.k8s.io}
---
apiVersion: batch/v1
kind: CronJob
metadata:
name: sa-cleaner
namespace: kube-system
spec:
schedule: "0 */6 * * *" # Run every 6 hours
jobTemplate:
spec:
template:
spec:
serviceAccountName: sa-cleaner
containers:
- name: cleaner
image: rancher/kubectl:v1.26.0
command:
- /bin/bash
- -c
- |
# Get all namespaces
for ns in $(kubectl get ns -o jsonpath='{.items[*].metadata.name}'); do
# Check if default SA has automount=true (or null)
automount=$(kubectl get sa default -n $ns -o jsonpath='{.automountServiceAccountToken}')
if [ "$automount" != "false" ]; then
echo "Securing default SA in namespace: $ns"
kubectl patch sa default -n $ns -p '{"automountServiceAccountToken": false}'
fi
done
restartPolicy: OnFailure
# ----------------------------------------------------------------
# 6. OS Sysctls Hardening
# ----------------------------------------------------------------
- path: /etc/sysctl.d/60-rke2-cis.conf
permissions: '0644'
content: |
vm.overcommit_memory=1
vm.max_map_count=65530
vm.panic_on_oom=0
fs.inotify.max_user_watches=1048576
fs.inotify.max_user_instances=8192
kernel.panic=10
kernel.panic_on_oops=1
net.ipv4.conf.all.rp_filter=1
net.ipv4.conf.default.rp_filter=1
net.ipv4.conf.all.accept_source_route=0
net.ipv4.conf.default.accept_source_route=0
net.ipv4.conf.all.accept_redirects=0
net.ipv4.conf.default.accept_redirects=0
net.ipv4.conf.all.send_redirects=0
net.ipv4.conf.default.send_redirects=0
net.ipv4.conf.all.log_martians=1
net.ipv4.conf.default.log_martians=1
net.ipv4.icmp_echo_ignore_broadcasts=1
net.ipv4.icmp_ignore_bogus_error_responses=1
net.ipv6.conf.all.disable_ipv6=1
net.ipv6.conf.default.disable_ipv6=1
fs.protected_hardlinks=1
fs.protected_symlinks=1
# ----------------------------------------------------------------
# 7. Environment & Setup Scripts
# ----------------------------------------------------------------
- path: /etc/profile.d/rke2.sh
permissions: '0644'
content: |
export PATH=$PATH:/var/lib/rancher/rke2/bin:/opt/rke2/bin
export KUBECONFIG=/etc/rancher/rke2/rke2.yaml
- path: /root/updates.sh
permissions: '0550'
content: |
#!/bin/bash
export DEBIAN_FRONTEND=noninteractive
apt-mark hold linux-headers-generic
apt-mark hold linux-headers-virtual
apt-mark hold linux-image-virtual
apt-mark hold linux-virtual
apt-get update
apt-get upgrade -y
apt-get autoremove -y
users:
- name: rancher
gecos: Rancher service account
hashed_passwd: $6$Mas.x2i7B2cefjUy$59363FmEuoU.LiTLNRZmtemlH2W0D0SWsig22KSZ3QzOmfxeZXxdSx5wIw9wO7GXF/M9W.9SHoKVBOYj1HPX3.
lock_passwd: false
shell: /bin/bash
groups: [users, sudo, docker]
sudo: ALL=(ALL:ALL) ALL
ssh_authorized_keys:
- 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEwWnnOTAu0LlAZRczQ0Z0KvNlUdPhGQhpZie+nF1O3s'
- name: etcd
gecos: "etcd user"
shell: /sbin/nologin
system: true
lock_passwd: true
disable_root: true
ssh_pwauth: true
runcmd:
- systemctl enable --now qemu-guest-agent
- sysctl --system
- /root/updates.sh
# Immediate run of fix script
- /usr/local/bin/fix-cni-perms.sh
final_message: |
VI_CNV_CLOUD_INIT has been applied successfully.
Node ready for Rancher!
# amazonec2, azure, digitalocean, harvester, vsphere, custom
cloudprovider: harvester
# cloud provider credentials
cloudCredentialSecretName: cc-mrklm
# rancher manager url
rancher:
cattle:
url: rancher-mgmt.product.lan
# cluster values
cluster:
name: default-cluster
# labels:
# key: value
config:
kubernetesVersion: v1.33.5+rke2r1
enableNetworkPolicy: true
localClusterAuthEndpoint:
enabled: false
chartValues:
harvester-cloud-provider:
global:
cattle:
clusterName: default-cluster
# Pod Security Standard (Replaces PSP)
defaultPodSecurityAdmissionConfigurationTemplateName: "rancher-restricted"
globalConfig:
systemDefaultRegistry: docker.io
cni: canal
docker: false
disable_scheduler: false
disable_cloud_controller: false
disable_kube_proxy: false
etcd_expose_metrics: false
profile: 'cis'
selinux: false
secrets_encryption: true
write_kubeconfig_mode: 0600
use_service_account_credentials: false
protect_kernel_defaults: true
cloud_provider_name: harvester
cloud_provider_config: secret://fleet-default:harvesterconfigzswmd
kube_apiserver_arg:
- "service-account-extend-token-expiration=false"
- "anonymous-auth=false"
- "enable-admission-plugins=NodeRestriction,PodSecurity,EventRateLimit,DenyServiceExternalIPs"
- "admission-control-config-file=/etc/rancher/rke2/rke2-admission.yaml"
- "audit-policy-file=/etc/rancher/rke2/audit-policy.yaml"
- "audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log"
- "audit-log-maxage=30"
- "audit-log-maxbackup=10"
- "audit-log-maxsize=100"
kubelet_arg:
# Strong Ciphers (CIS 4.2.12)
- "tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305"
# PID Limit (CIS 4.2.13)
- "pod-max-pids=4096"
# Seccomp Default (CIS 4.2.14)
- "seccomp-default=true"
- "protect-kernel-defaults=true"
- "make-iptables-util-chains=true"
upgradeStrategy:
controlPlaneConcurrency: 10%
controlPlaneDrainOptions:
enabled: false
workerConcurrency: 10%
workerDrainOptions:
enabled: false
addons:
monitoring:
enabled: false
logging:
enabled: false
longhorn:
enabled: false
neuvector:
enabled: false
# node and nodepool(s) values
# ----------------------------------------------------------------
# MANUAL TESTING SECTION
# The Operator will DELETE and OVERWRITE this section at runtime.
# These values are only used if you run 'helm install' manually.
# ----------------------------------------------------------------
nodepools:
- name: control-plane-nodes
displayName: cp-nodes
quantity: 1
etcd: true
controlplane: true
worker: false
paused: false
cpuCount: 4
diskSize: 40
imageName: vanderlande/image-qhtpc
memorySize: 8
networkName: vanderlande/vm-lan
sshUser: rancher
vmNamespace: vanderlande
userData: *userData
- name: worker-nodes
displayName: wk-nodes
quantity: 2
etcd: false
controlplane: false
worker: true
paused: false
cpuCount: 2
diskSize: 40
imageName: vanderlande/image-qmx5q
memorySize: 8
networkName: vanderlande/vm-lan
sshUser: rancher
vmNamespace: vanderlande
userData: *userData

205
deploy/k8s-provisioner/internal/templates/base_values_vsphere.yaml Normal file
View File

@@ -0,0 +1,205 @@
# ----------------------------------------------------------------
# BASE TEMPLATE (internal/templates/base_values.yaml)
# ----------------------------------------------------------------
_defaults:
helmChart:
repo: ""
name: "oci://ghcr.io/rancherfederal/charts/rancher-cluster-templates"
version: "0.7.2"
controlPlaneProfile:
cpuCores: 4
memoryGb: 8
diskGb: 40
userData: &userData |
#cloud-config
package_update: false
package_upgrade: false
snap:
commands:
00: snap refresh --hold=forever
package_reboot_if_required: true
packages:
- yq
- jq
disable_root: true
ssh_pwauth: false
write_files:
- path: /root/updates.sh
permissions: '0550'
content: |
#!/bin/bash
export DEBIAN_FRONTEND=noninteractive
apt-mark hold linux-headers-generic
apt-mark hold linux-headers-virtual
apt-mark hold linux-image-virtual
apt-mark hold linux-virtual
apt-get update
apt-get upgrade -y
apt-get autoremove -y
users:
- name: rancher
gecos: Rancher service account
hashed_passwd: $6$Mas.x2i7B2cefjUy$59363FmEuoU.LiTLNRZmtemlH2W0D0SWsig22KSZ3QzOmfxeZXxdSx5wIw9wO7GXF/M9W.9SHoKVBOYj1HPX3.
lock_passwd: false
shell: /bin/bash
groups: [users, sudo, docker]
sudo: ALL=(ALL:ALL) ALL
ssh_authorized_keys:
- 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEwWnnOTAu0LlAZRczQ0Z0KvNlUdPhGQhpZie+nF1O3s'
disable_root: true
ssh_pwauth: true
runcmd:
# - systemctl enable --now qemu-guest-agent
- sysctl --system
- /root/updates.sh
# Immediate run of fix script
bootcmd:
- sudo bash /root/networking.sh
final_message: |
VI_CNV_CLOUD_INIT has been applied successfully.
Node ready for Rancher!
# amazonec2, azure, digitalocean, harvester, vsphere, custom
cloudprovider: vsphere
# cloud provider credentials
cloudCredentialSecretName: cc-lhtl9
# rancher manager url
rancher:
cattle:
url: rancher.tst.vanderlande.com
# cluster values
cluster:
name: default-cluster-005
# labels:
# key: value
config:
kubernetesVersion: v1.31.12+rke2r1
enableNetworkPolicy: true
localClusterAuthEndpoint:
enabled: false
# Pod Security Standard (Replaces PSP)
# defaultPodSecurityAdmissionConfigurationTemplateName: "rancher-restricted"
globalConfig:
systemDefaultRegistry: docker.io
cni: canal
docker: false
disable_scheduler: false
disable_cloud_controller: false
disable_kube_proxy: false
etcd_expose_metrics: false
profile: ''
selinux: false
secrets_encryption: false
write_kubeconfig_mode: 0600
use_service_account_credentials: false
protect_kernel_defaults: false
cloud_provider_name: ''
# kube_apiserver_arg:
# - "service-account-extend-token-expiration=false"
# - "anonymous-auth=false"
# - "enable-admission-plugins=NodeRestriction,PodSecurity,EventRateLimit,DenyServiceExternalIPs"
# - "admission-control-config-file=/etc/rancher/rke2/rke2-admission.yaml"
# - "audit-policy-file=/etc/rancher/rke2/audit-policy.yaml"
# - "audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log"
# - "audit-log-maxage=30"
# - "audit-log-maxbackup=10"
# - "audit-log-maxsize=100"
# kubelet_arg:
# # Strong Ciphers (CIS 4.2.12)
# - "tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305"
# # PID Limit (CIS 4.2.13)
# - "pod-max-pids=4096"
# # Seccomp Default (CIS 4.2.14)
# - "seccomp-default=true"
# - "protect-kernel-defaults=true"
# - "make-iptables-util-chains=true"
upgradeStrategy:
controlPlaneConcurrency: 10%
controlPlaneDrainOptions:
enabled: false
workerConcurrency: 10%
workerDrainOptions:
enabled: false
addons:
monitoring:
enabled: false
logging:
enabled: false
longhorn:
enabled: true
neuvector:
enabled: false
# node and nodepool(s) values
# ----------------------------------------------------------------
# MANUAL TESTING SECTION
# The Operator will DELETE and OVERWRITE this section at runtime.
# These values are only used if you run 'helm install' manually.
# ----------------------------------------------------------------
nodepools:
- name: control-plane-nodes
displayName: cp-nodes
quantity: 1
etcd: true
controlplane: true
worker: false
paused: false
# VSPHERE SPECIFIC FIELDS
cpuCount: 2
memorySize: 8192
diskSize: 40000
vcenter: "vcenter.vanderlande.com"
datacenter: "NL001"
folder: "ICT Digitalisation - Rancher"
pool: "NL001 Development - Rancher/Resources"
datastoreCluster: "NL001 Development - Rancher SDRS" # Matches your SDRS input
network:
- "nl001.vDS.Distri.Vlan.1542"
# Provisioning Source
creationType: "template"
cloneFrom: "nl001-cp-ubuntu-22.04-amd64-20250327-5.15.0-135-rke2-k3s"
cloudConfig: *userData # Using the anchor from your base file
- name: worker-storage-nodes
displayName: wk-nodes
quantity: 2
etcd: false
controlplane: false
worker: true
paused: false
# VSPHERE SPECIFIC FIELDS
cpuCount: 4
memorySize: 8192
diskSize: 100000
vcenter: "vcenter.vanderlande.com"
datacenter: "NL001"
folder: "ICT Digitalisation - Rancher"
pool: "NL001 Development - Rancher/Resources"
datastoreCluster: "NL001 Development - Rancher SDRS" # Matches your SDRS input
network:
- "nl001.vDS.Distri.Vlan.1542"
# Provisioning Source
creationType: "template"
cloneFrom: "nl001-cp-ubuntu-22.04-amd64-20250327-5.15.0-135-rke2-k3s"
cloudConfig: *userData # Using the anchor from your base file

6
deploy/k8s-provisioner/internal/templates/embed.go Normal file
View File

@@ -0,0 +1,6 @@
package templates
import _ "embed"
//go:embed base_values.yaml
var BaseValuesYAML []byte