Go.Rig-Operator/deploy/rig-operator/internal/provider/harvester/credential.go

package harvester

import (
	"context"
	"encoding/base64"
	"fmt"
	"time"

	authenticationv1 "k8s.io/api/authentication/v1"
	corev1 "k8s.io/api/core/v1"
	rbacv1 "k8s.io/api/rbac/v1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/tools/clientcmd"
)

// DeleteCredentialResources connects to Harvester and removes the specific SA and bindings
func DeleteCredentialResources(ctx context.Context, masterKubeconfig []byte, serviceAccountName, vmNamespace string) error {
	restConfig, err := clientcmd.RESTConfigFromKubeConfig(masterKubeconfig)
	if err != nil {
		return err
	}
	hvClient, err := kubernetes.NewForConfig(restConfig)
	if err != nil {
		return err
	}

	deletePolicy := metav1.DeletePropagationBackground
	deleteOpts := metav1.DeleteOptions{PropagationPolicy: &deletePolicy}

	// 1. Delete Global CSI Binding (ClusterRoleBinding)
	csiBindingName := fmt.Sprintf("%s-csi-binding", serviceAccountName)
	// We ignore NotFound errors to make this idempotent
	if err := hvClient.RbacV1().ClusterRoleBindings().Delete(ctx, csiBindingName, deleteOpts); err != nil && !apierrors.IsNotFound(err) {
		return err
	}

	// 2. Delete Cloud Provider Binding (RoleBinding in VM Namespace)
	cpBindingName := fmt.Sprintf("%s-cloud-binding", serviceAccountName)
	if err := hvClient.RbacV1().RoleBindings(vmNamespace).Delete(ctx, cpBindingName, deleteOpts); err != nil && !apierrors.IsNotFound(err) {
		return err
	}

	// 3. Delete ServiceAccount (VM Namespace)
	if err := hvClient.CoreV1().ServiceAccounts(vmNamespace).Delete(ctx, serviceAccountName, deleteOpts); err != nil && !apierrors.IsNotFound(err) {
		return err
	}

	return nil
}

// EnsureCredential mints a dedicated ServiceAccount in the specific VM Namespace
func EnsureCredential(ctx context.Context, masterKubeconfig []byte, clusterName, targetNamespace, vmNamespace, harvesterURL string) (*corev1.Secret, string, time.Time, error) {

	// --- PHASE 1: Connect ---
	restConfig, err := clientcmd.RESTConfigFromKubeConfig(masterKubeconfig)
	if err != nil {
		return nil, "", time.Time{}, fmt.Errorf("invalid rancher cloud credential kubeconfig: %w", err)
	}
	hvClient, err := kubernetes.NewForConfig(restConfig)
	if err != nil {
		return nil, "", time.Time{}, err
	}

	// --- PHASE 2: Create Identity ---
	if vmNamespace == "" {
		vmNamespace = "default"
	}
	saName := fmt.Sprintf("prov-%s", clusterName)

	// A. Create ServiceAccount
	sa := &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: saName, Namespace: vmNamespace}}
	if _, err := hvClient.CoreV1().ServiceAccounts(vmNamespace).Create(ctx, sa, metav1.CreateOptions{}); err != nil {
		if !apierrors.IsAlreadyExists(err) {
			return nil, "", time.Time{}, err
		}
	}

	// B. Create RoleBinding (Cloud Provider)
	rb := &rbacv1.RoleBinding{
		ObjectMeta: metav1.ObjectMeta{Name: saName + "-cloud-binding", Namespace: vmNamespace},
		Subjects:   []rbacv1.Subject{{Kind: "ServiceAccount", Name: saName, Namespace: vmNamespace}},
		RoleRef:    rbacv1.RoleRef{Kind: "ClusterRole", Name: "harvesterhci.io:cloudprovider", APIGroup: "rbac.authorization.k8s.io"},
	}
	if _, err := hvClient.RbacV1().RoleBindings(vmNamespace).Create(ctx, rb, metav1.CreateOptions{}); err != nil {
		if !apierrors.IsAlreadyExists(err) { /* Ignore */
		}
	}

	// C. Create ClusterRoleBinding (CSI Driver)
	crb := &rbacv1.ClusterRoleBinding{
		ObjectMeta: metav1.ObjectMeta{Name: saName + "-csi-binding"},
		Subjects:   []rbacv1.Subject{{Kind: "ServiceAccount", Name: saName, Namespace: vmNamespace}},
		RoleRef:    rbacv1.RoleRef{Kind: "ClusterRole", Name: "harvesterhci.io:csi-driver", APIGroup: "rbac.authorization.k8s.io"},
	}
	if _, err := hvClient.RbacV1().ClusterRoleBindings().Create(ctx, crb, metav1.CreateOptions{}); err != nil {
		if !apierrors.IsAlreadyExists(err) { /* Ignore */
		}
	}

	// D. Mint Token
	ttlSeconds := int64(315360000) // ~10 years
	tokenRequest, err := hvClient.CoreV1().ServiceAccounts(vmNamespace).CreateToken(ctx, saName, &authenticationv1.TokenRequest{
		Spec: authenticationv1.TokenRequestSpec{ExpirationSeconds: &ttlSeconds},
	}, metav1.CreateOptions{})
	if err != nil {
		return nil, "", time.Time{}, fmt.Errorf("failed to mint harvester token: %w", err)
	}
	expiryTime := time.Now().Add(time.Duration(ttlSeconds) * time.Second)

	// --- PHASE 3: Determine URL & CA ---
	if harvesterURL == "" {
		harvesterURL = restConfig.Host
	}

	// Fetch internal CA (required because proxy CA != internal CA)
	harvesterCA := restConfig.CAData
	caConfigMap, err := hvClient.CoreV1().ConfigMaps("default").Get(ctx, "kube-root-ca.crt", metav1.GetOptions{})
	if err == nil {
		if caStr, ok := caConfigMap.Data["ca.crt"]; ok {
			harvesterCA = []byte(caStr)
		}
	}

	// --- PHASE 4: Construct Kubeconfig ---
	caData := base64.StdEncoding.EncodeToString(harvesterCA)
	token := tokenRequest.Status.Token

	newKubeconfig := fmt.Sprintf(
		`apiVersion: v1
kind: Config
clusters:
- name: harvester
  cluster:
    server: %s
    certificate-authority-data: %s
users:
- name: provisioner
  user:
    token: %s
contexts:
- name: default
  context:
    cluster: harvester
    user: provisioner
    namespace: %s
current-context: default
`, harvesterURL, caData, token, vmNamespace)

	// --- PHASE 5: Create Secret Object ---
	secretName := fmt.Sprintf("harvesterconfig-%s", clusterName)

	secret := &corev1.Secret{
		TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"},
		ObjectMeta: metav1.ObjectMeta{
			Name:      secretName,
			Namespace: targetNamespace,
			Annotations: map[string]string{
				// [CRITICAL] These annotations authorize the guest cluster to use this secret
				"v2prov-secret-authorized-for-cluster":                clusterName,
				"v2prov-authorized-secret-deletes-on-cluster-removal": "true",
			},
			Labels: map[string]string{
				"cattle.io/creator":       "rig-operator", // Updated creator
				"rig.appstack.io/cluster": clusterName,
			},
		},
		Type: "Opaque",
		StringData: map[string]string{
			"credential": newKubeconfig,
		},
	}

	return secret, saName, expiryTime, nil
}