Go.Rig-Operator/deploy/k8s-provisioner/internal/harvester/factory.go

package harvester

import (
	"context"
	"encoding/base64"
	"fmt"
	"time"

	authenticationv1 "k8s.io/api/authentication/v1"
	corev1 "k8s.io/api/core/v1"
	rbacv1 "k8s.io/api/rbac/v1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/tools/clientcmd"
	"sigs.k8s.io/controller-runtime/pkg/client"
	"sigs.k8s.io/controller-runtime/pkg/log"

	k8sprovisionerv1alpha1 "vanderlande.com/appstack/k8s-provisioner/api/v1alpha1"
)

// TryCleanup performs a "Best Effort" cleanup of Harvester resources.
func TryCleanup(ctx context.Context, k8sClient client.Client, infraRefName, namespace, saName string) {
	l := log.FromContext(ctx)

	// 1. Fetch Infra
	var infra k8sprovisionerv1alpha1.Infra
	if err := k8sClient.Get(ctx, types.NamespacedName{Name: infraRefName, Namespace: namespace}, &infra); err != nil {
		l.Info("Cleanup skipped: Infra object not found")
		return
	}

	vmNamespace := infra.Spec.VmNamespace
	if vmNamespace == "" {
		vmNamespace = "default"
	}

	// 2. Fetch Master Credential
	rancherCredName := infra.Spec.CloudCredentialSecret
	var rancherSecret corev1.Secret
	if err := k8sClient.Get(ctx, types.NamespacedName{Name: rancherCredName, Namespace: "cattle-global-data"}, &rancherSecret); err != nil {
		l.Info("Cleanup skipped: Master Credential Secret not found")
		return
	}

	// 3. Extract Kubeconfig
	var kubeBytes []byte
	if len(rancherSecret.Data["harvestercredentialConfig-kubeconfigContent"]) > 0 {
		kubeBytes = rancherSecret.Data["harvestercredentialConfig-kubeconfigContent"]
	} else if len(rancherSecret.Data["credential"]) > 0 {
		kubeBytes = rancherSecret.Data["credential"]
	} else {
		return
	}

	// 4. Cleanup
	if err := deleteHarvesterResources(ctx, kubeBytes, saName, vmNamespace); err != nil {
		l.Error(err, "Failed to cleanup Harvester resources (ignoring)")
	} else {
		l.Info("Harvester resources deleted successfully")
	}
}

// Internal helper for cleanup
func deleteHarvesterResources(ctx context.Context, masterKubeconfig []byte, serviceAccountName, vmNamespace string) error {
	restConfig, err := clientcmd.RESTConfigFromKubeConfig(masterKubeconfig)
	if err != nil {
		return err
	}
	hvClient, err := kubernetes.NewForConfig(restConfig)
	if err != nil {
		return err
	}

	deletePolicy := metav1.DeletePropagationBackground
	deleteOpts := metav1.DeleteOptions{PropagationPolicy: &deletePolicy}

	// 1. Delete Global CSI Binding (ClusterRoleBinding)
	csiBindingName := fmt.Sprintf("%s-csi-binding", serviceAccountName)
	err = hvClient.RbacV1().ClusterRoleBindings().Delete(ctx, csiBindingName, deleteOpts)
	if err != nil && !apierrors.IsNotFound(err) {
		return err
	}

	// 2. Delete Cloud Provider Binding (RoleBinding in VM Namespace)
	cpBindingName := fmt.Sprintf("%s-cloud-binding", serviceAccountName)
	err = hvClient.RbacV1().RoleBindings(vmNamespace).Delete(ctx, cpBindingName, deleteOpts)
	if err != nil && !apierrors.IsNotFound(err) {
		return err
	}

	// 3. Delete ServiceAccount (VM Namespace)
	err = hvClient.CoreV1().ServiceAccounts(vmNamespace).Delete(ctx, serviceAccountName, deleteOpts)
	if err != nil && !apierrors.IsNotFound(err) {
		return err
	}

	return nil
}

// EnsureCredential mints a dedicated ServiceAccount in the specific VM Namespace
func EnsureCredential(ctx context.Context, masterKubeconfig []byte, clusterName, targetNamespace, vmNamespace, harvesterURL string) (*corev1.Secret, string, time.Time, error) {

	// --- PHASE 1: Connect (Proxy/Master Config) ---
	restConfig, err := clientcmd.RESTConfigFromKubeConfig(masterKubeconfig)
	if err != nil {
		return nil, "", time.Time{}, fmt.Errorf("invalid rancher cloud credential kubeconfig: %w", err)
	}
	hvClient, err := kubernetes.NewForConfig(restConfig)
	if err != nil {
		return nil, "", time.Time{}, err
	}

	// --- PHASE 2: Create Identity (SA & Bindings) ---
	if vmNamespace == "" {
		vmNamespace = "default"
	}
	saName := fmt.Sprintf("prov-%s", clusterName)

	// A. Create ServiceAccount
	sa := &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: saName, Namespace: vmNamespace}}
	if _, err := hvClient.CoreV1().ServiceAccounts(vmNamespace).Create(ctx, sa, metav1.CreateOptions{}); err != nil {
		if !apierrors.IsAlreadyExists(err) {
			return nil, "", time.Time{}, err
		}
	}

	// B. Create RoleBinding (VM Namespace)
	rb := &rbacv1.RoleBinding{
		ObjectMeta: metav1.ObjectMeta{Name: saName + "-cloud-binding", Namespace: vmNamespace},
		Subjects:   []rbacv1.Subject{{Kind: "ServiceAccount", Name: saName, Namespace: vmNamespace}},
		RoleRef:    rbacv1.RoleRef{Kind: "ClusterRole", Name: "harvesterhci.io:cloudprovider", APIGroup: "rbac.authorization.k8s.io"},
	}
	if _, err := hvClient.RbacV1().RoleBindings(vmNamespace).Create(ctx, rb, metav1.CreateOptions{}); err != nil {
		if !apierrors.IsAlreadyExists(err) { /* Ignore */
		}
	}

	// C. Create ClusterRoleBinding (Global)
	crb := &rbacv1.ClusterRoleBinding{
		ObjectMeta: metav1.ObjectMeta{Name: saName + "-csi-binding"},
		Subjects:   []rbacv1.Subject{{Kind: "ServiceAccount", Name: saName, Namespace: vmNamespace}},
		RoleRef:    rbacv1.RoleRef{Kind: "ClusterRole", Name: "harvesterhci.io:csi-driver", APIGroup: "rbac.authorization.k8s.io"},
	}
	if _, err := hvClient.RbacV1().ClusterRoleBindings().Create(ctx, crb, metav1.CreateOptions{}); err != nil {
		if !apierrors.IsAlreadyExists(err) { /* Ignore */
		}
	}

	// D. Mint Token
	ttlSeconds := int64(315360000)
	tokenRequest, err := hvClient.CoreV1().ServiceAccounts(vmNamespace).CreateToken(ctx, saName, &authenticationv1.TokenRequest{
		Spec: authenticationv1.TokenRequestSpec{ExpirationSeconds: &ttlSeconds},
	}, metav1.CreateOptions{})
	if err != nil {
		return nil, "", time.Time{}, fmt.Errorf("failed to mint harvester token: %w", err)
	}
	expiryTime := time.Now().Add(time.Duration(ttlSeconds) * time.Second)

	// --- PHASE 3: Determine URL & CA ---

	// 1. URL: Use the explicitly provided HarvesterURL
	if harvesterURL == "" {
		// Fallback to Proxy if user forgot to set it (Safety net)
		harvesterURL = restConfig.Host
	}

	// 2. CA: Fetch the internal Harvester CA
	// (Required because the proxy CA won't match the direct IP/URL)
	harvesterCA := restConfig.CAData

	caConfigMap, err := hvClient.CoreV1().ConfigMaps("default").Get(ctx, "kube-root-ca.crt", metav1.GetOptions{})
	if err == nil {
		if caStr, ok := caConfigMap.Data["ca.crt"]; ok {
			harvesterCA = []byte(caStr)
		}
	}

	// --- PHASE 4: Construct Kubeconfig ---
	caData := base64.StdEncoding.EncodeToString(harvesterCA)
	token := tokenRequest.Status.Token

	// Ensure "namespace" aligns vertically with "cluster" and "user"
	newKubeconfig := fmt.Sprintf(
		`apiVersion: v1
kind: Config
clusters:
- name: harvester
  cluster:
    server: %s
    certificate-authority-data: %s
users:
- name: provisioner
  user:
    token: %s
contexts:
- name: default
  context:
    cluster: harvester
    user: provisioner
    namespace: %s
current-context: default
`, harvesterURL, caData, token, vmNamespace)

	// --- PHASE 5: Create Secret ---
	secretName := fmt.Sprintf("harvesterconfig-%s", clusterName)

	secret := &corev1.Secret{
		TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"},
		ObjectMeta: metav1.ObjectMeta{
			Name:      secretName,
			Namespace: targetNamespace,
			Annotations: map[string]string{
				"v2prov-secret-authorized-for-cluster":                clusterName,
				"v2prov-authorized-secret-deletes-on-cluster-removal": "true",
			},
			Labels: map[string]string{
				"cattle.io/creator": "k8s-provisioner",
			},
		},
		Type: "Opaque",
		StringData: map[string]string{
			"credential": newKubeconfig,
		},
	}

	return secret, saName, expiryTime, nil
}