ContainerImage.Pinniped/internal/controllermanager/prepare_controllers.go

// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved.
// SPDX-License-Identifier: Apache-2.0

// Package controllermanager provides an entrypoint into running all of the controllers that run as
// a part of Pinniped.
package controllermanager

import (
	"context"
	"crypto/tls"
	"fmt"
	"net/http"
	"time"

	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/util/clock"
	k8sinformers "k8s.io/client-go/informers"
	"k8s.io/client-go/kubernetes"
	"k8s.io/klog/v2/klogr"

	loginv1alpha1 "go.pinniped.dev/generated/1.20/apis/concierge/login/v1alpha1"
	pinnipedclientset "go.pinniped.dev/generated/1.20/client/concierge/clientset/versioned"
	pinnipedinformers "go.pinniped.dev/generated/1.20/client/concierge/informers/externalversions"
	"go.pinniped.dev/internal/apiserviceref"
	"go.pinniped.dev/internal/concierge/impersonator"
	"go.pinniped.dev/internal/config/concierge"
	"go.pinniped.dev/internal/controller/apicerts"
	"go.pinniped.dev/internal/controller/authenticator/authncache"
	"go.pinniped.dev/internal/controller/authenticator/cachecleaner"
	"go.pinniped.dev/internal/controller/authenticator/jwtcachefiller"
	"go.pinniped.dev/internal/controller/authenticator/webhookcachefiller"
	"go.pinniped.dev/internal/controller/impersonatorconfig"
	"go.pinniped.dev/internal/controller/issuerconfig"
	"go.pinniped.dev/internal/controller/kubecertagent"
	"go.pinniped.dev/internal/controllerlib"
	"go.pinniped.dev/internal/deploymentref"
	"go.pinniped.dev/internal/downward"
	"go.pinniped.dev/internal/dynamiccert"
	"go.pinniped.dev/internal/groupsuffix"
	"go.pinniped.dev/internal/kubeclient"
)

const (
	singletonWorker       = 1
	defaultResyncInterval = 3 * time.Minute
)

// Config holds all the input parameters to the set of controllers run as a part of Pinniped.
//
// It is used to inject parameters into PrepareControllers.
type Config struct {
	// ServerInstallationInfo provides the name of the pod in which Pinniped is running and the namespace in which Pinniped is deployed.
	ServerInstallationInfo *downward.PodInfo

	// APIGroupSuffix is the suffix of the Pinniped API that should be targeted by these controllers.
	APIGroupSuffix string

	// NamesConfig comes from the Pinniped config API (see api.Config). It specifies how Kubernetes
	// objects should be named.
	NamesConfig *concierge.NamesConfigSpec

	// KubeCertAgentConfig comes from the Pinniped config API (see api.Config). It configures how
	// the kubecertagent package's controllers should manage the agent pods.
	KubeCertAgentConfig *concierge.KubeCertAgentSpec

	// DiscoveryURLOverride allows a caller to inject a hardcoded discovery URL into Pinniped
	// discovery document.
	DiscoveryURLOverride *string

	// DynamicServingCertProvider provides a setter and a getter to the Pinniped API's serving cert.
	DynamicServingCertProvider dynamiccert.Provider
	// DynamicSigningCertProvider provides a setter and a getter to the Pinniped API's
	// signing cert, i.e., the cert that it uses to sign certs for Pinniped clients wishing to login.
	DynamicSigningCertProvider dynamiccert.Provider

	// ServingCertDuration is the validity period, in seconds, of the API serving certificate.
	ServingCertDuration time.Duration
	// ServingCertRenewBefore is the period of time, in seconds, that pinniped will wait before
	// rotating the serving certificate. This period of time starts upon issuance of the serving
	// certificate.
	ServingCertRenewBefore time.Duration

	// AuthenticatorCache is a cache of authenticators shared amongst various authenticated-related controllers.
	AuthenticatorCache *authncache.Cache

	// LoginJSONDecoder can decode login.concierge.pinniped.dev types (e.g., TokenCredentialRequest)
	// into their internal representation.
	LoginJSONDecoder runtime.Decoder

	// Labels are labels that should be added to any resources created by the controllers.
	Labels map[string]string
}

// Prepare the controllers and their informers and return a function that will start them when called.
//nolint:funlen // Eh, fair, it is a really long function...but it is wiring the world...so...
func PrepareControllers(c *Config) (func(ctx context.Context), error) {
	groupName, ok := groupsuffix.Replace(loginv1alpha1.GroupName, c.APIGroupSuffix)
	if !ok {
		return nil, fmt.Errorf("cannot make api group from %s/%s", loginv1alpha1.GroupName, c.APIGroupSuffix)
	}
	apiServiceName := loginv1alpha1.SchemeGroupVersion.Version + "." + groupName

	dref, _, err := deploymentref.New(c.ServerInstallationInfo)
	if err != nil {
		return nil, fmt.Errorf("cannot create deployment ref: %w", err)
	}

	apiServiceRef, err := apiserviceref.New(apiServiceName)
	if err != nil {
		return nil, fmt.Errorf("cannot create API service ref: %w", err)
	}

	client, err := kubeclient.New(
		dref,          // first try to use the deployment as an owner ref (for namespace scoped resources)
		apiServiceRef, // fallback to our API service (for everything else we create)
		kubeclient.WithMiddleware(groupsuffix.New(c.APIGroupSuffix)),
	)
	if err != nil {
		return nil, fmt.Errorf("could not create clients for the controllers: %w", err)
	}

	// Create informers. Don't forget to make sure they get started in the function returned below.
	informers := createInformers(c.ServerInstallationInfo.Namespace, client.Kubernetes, client.PinnipedConcierge)

	// Configuration for the kubecertagent controllers created below.
	agentPodConfig := &kubecertagent.AgentPodConfig{
		Namespace:                 c.ServerInstallationInfo.Namespace,
		ContainerImage:            *c.KubeCertAgentConfig.Image,
		PodNamePrefix:             *c.KubeCertAgentConfig.NamePrefix,
		ContainerImagePullSecrets: c.KubeCertAgentConfig.ImagePullSecrets,
		AdditionalLabels:          c.Labels,
	}
	credentialIssuerLocationConfig := &kubecertagent.CredentialIssuerLocationConfig{
		Name: c.NamesConfig.CredentialIssuer,
	}

	// Create controller manager.
	controllerManager := controllerlib.
		NewManager().

		// KubeConfig info publishing controller is responsible for writing the KubeConfig information to the
		// CredentialIssuer resource and keeping that information up to date.
		WithController(
			issuerconfig.NewKubeConfigInfoPublisherController(
				c.NamesConfig.CredentialIssuer,
				c.Labels,
				c.DiscoveryURLOverride,
				client.PinnipedConcierge,
				informers.kubePublicNamespaceK8s.Core().V1().ConfigMaps(),
				controllerlib.WithInformer,
			),
			singletonWorker,
		).

		// API certs controllers are responsible for managing the TLS certificates used to serve Pinniped's API.
		WithController(
			apicerts.NewCertsManagerController(
				c.ServerInstallationInfo.Namespace,
				c.NamesConfig.ServingCertificateSecret,
				c.Labels,
				client.Kubernetes,
				informers.installationNamespaceK8s.Core().V1().Secrets(),
				controllerlib.WithInformer,
				controllerlib.WithInitialEvent,
				c.ServingCertDuration,
				"Pinniped CA",
				c.NamesConfig.APIService,
			),
			singletonWorker,
		).
		WithController(
			apicerts.NewAPIServiceUpdaterController(
				c.ServerInstallationInfo.Namespace,
				c.NamesConfig.ServingCertificateSecret,
				apiServiceName,
				client.Aggregation,
				informers.installationNamespaceK8s.Core().V1().Secrets(),
				controllerlib.WithInformer,
			),
			singletonWorker,
		).
		WithController(
			apicerts.NewCertsObserverController(
				c.ServerInstallationInfo.Namespace,
				c.NamesConfig.ServingCertificateSecret,
				c.DynamicServingCertProvider,
				informers.installationNamespaceK8s.Core().V1().Secrets(),
				controllerlib.WithInformer,
			),
			singletonWorker,
		).
		WithController(
			apicerts.NewCertsExpirerController(
				c.ServerInstallationInfo.Namespace,
				c.NamesConfig.ServingCertificateSecret,
				client.Kubernetes,
				informers.installationNamespaceK8s.Core().V1().Secrets(),
				controllerlib.WithInformer,
				c.ServingCertRenewBefore,
			),
			singletonWorker,
		).

		// Kube cert agent controllers are responsible for finding the cluster's signing keys and keeping them
		// up to date in memory, as well as reporting status on this cluster integration strategy.
		WithController(
			kubecertagent.NewCreaterController(
				agentPodConfig,
				credentialIssuerLocationConfig,
				c.Labels,
				clock.RealClock{},
				client.Kubernetes,
				client.PinnipedConcierge,
				informers.kubeSystemNamespaceK8s.Core().V1().Pods(),
				informers.installationNamespaceK8s.Core().V1().Pods(),
				controllerlib.WithInformer,
				controllerlib.WithInitialEvent,
			),
			singletonWorker,
		).
		WithController(
			kubecertagent.NewAnnotaterController(
				agentPodConfig,
				credentialIssuerLocationConfig,
				clock.RealClock{},
				client.Kubernetes,
				client.PinnipedConcierge,
				informers.kubeSystemNamespaceK8s.Core().V1().Pods(),
				informers.installationNamespaceK8s.Core().V1().Pods(),
				controllerlib.WithInformer,
			),
			singletonWorker,
		).
		WithController(
			kubecertagent.NewExecerController(
				credentialIssuerLocationConfig,
				c.DynamicSigningCertProvider,
				kubecertagent.NewPodCommandExecutor(client.JSONConfig, client.Kubernetes),
				client.PinnipedConcierge,
				clock.RealClock{},
				informers.installationNamespaceK8s.Core().V1().Pods(),
				controllerlib.WithInformer,
			),
			singletonWorker,
		).
		WithController(
			kubecertagent.NewDeleterController(
				agentPodConfig,
				client.Kubernetes,
				informers.kubeSystemNamespaceK8s.Core().V1().Pods(),
				informers.installationNamespaceK8s.Core().V1().Pods(),
				controllerlib.WithInformer,
			),
			singletonWorker,
		).

		// The cache filler/cleaner controllers are responsible for keep an in-memory representation of active
		// authenticators up to date.
		WithController(
			webhookcachefiller.New(
				c.AuthenticatorCache,
				informers.pinniped.Authentication().V1alpha1().WebhookAuthenticators(),
				klogr.New(),
			),
			singletonWorker,
		).
		WithController(
			jwtcachefiller.New(
				c.AuthenticatorCache,
				informers.pinniped.Authentication().V1alpha1().JWTAuthenticators(),
				klogr.New(),
			),
			singletonWorker,
		).
		WithController(
			cachecleaner.New(
				c.AuthenticatorCache,
				informers.pinniped.Authentication().V1alpha1().WebhookAuthenticators(),
				informers.pinniped.Authentication().V1alpha1().JWTAuthenticators(),
				klogr.New(),
			),
			singletonWorker,
		).

		// The impersonation proxy configuration controllers dynamically configure the impersonation proxy feature.
		WithController(
			impersonatorconfig.NewImpersonatorConfigController(
				c.ServerInstallationInfo.Namespace,
				"pinniped-concierge-impersonation-proxy-config", // TODO this string should come from `c.NamesConfig`
				client.Kubernetes,
				informers.installationNamespaceK8s.Core().V1().ConfigMaps(),
				informers.installationNamespaceK8s.Core().V1().Services(),
				controllerlib.WithInformer,
				controllerlib.WithInitialEvent,
				"pinniped-concierge-impersonation-proxy-load-balancer", // TODO this string should come from `c.NamesConfig`
				c.Labels,
				tls.Listen,
				func() (http.Handler, error) {
					impersonationProxyHandler, err := impersonator.New(c.AuthenticatorCache, c.LoginJSONDecoder, klogr.New().WithName("impersonation-proxy"))
					if err != nil {
						return nil, fmt.Errorf("could not create impersonation proxy: %w", err)
					}
					return impersonationProxyHandler, nil
				},
			),
			singletonWorker,
		)

	// Return a function which starts the informers and controllers.
	return func(ctx context.Context) {
		informers.startAndWaitForSync(ctx)
		go controllerManager.Start(ctx)
	}, nil
}

type informers struct {
	kubePublicNamespaceK8s   k8sinformers.SharedInformerFactory
	kubeSystemNamespaceK8s   k8sinformers.SharedInformerFactory
	installationNamespaceK8s k8sinformers.SharedInformerFactory
	pinniped                 pinnipedinformers.SharedInformerFactory
}

// Create the informers that will be used by the controllers.
func createInformers(
	serverInstallationNamespace string,
	k8sClient kubernetes.Interface,
	pinnipedClient pinnipedclientset.Interface,
) *informers {
	return &informers{
		kubePublicNamespaceK8s: k8sinformers.NewSharedInformerFactoryWithOptions(
			k8sClient,
			defaultResyncInterval,
			k8sinformers.WithNamespace(issuerconfig.ClusterInfoNamespace),
		),
		kubeSystemNamespaceK8s: k8sinformers.NewSharedInformerFactoryWithOptions(
			k8sClient,
			defaultResyncInterval,
			k8sinformers.WithNamespace(kubecertagent.ControllerManagerNamespace),
		),
		installationNamespaceK8s: k8sinformers.NewSharedInformerFactoryWithOptions(
			k8sClient,
			defaultResyncInterval,
			k8sinformers.WithNamespace(serverInstallationNamespace),
		),
		pinniped: pinnipedinformers.NewSharedInformerFactoryWithOptions(
			pinnipedClient,
			defaultResyncInterval,
		),
	}
}

func (i *informers) startAndWaitForSync(ctx context.Context) {
	i.kubePublicNamespaceK8s.Start(ctx.Done())
	i.kubeSystemNamespaceK8s.Start(ctx.Done())
	i.installationNamespaceK8s.Start(ctx.Done())
	i.pinniped.Start(ctx.Done())

	i.kubePublicNamespaceK8s.WaitForCacheSync(ctx.Done())
	i.kubeSystemNamespaceK8s.WaitForCacheSync(ctx.Done())
	i.installationNamespaceK8s.WaitForCacheSync(ctx.Done())
	i.pinniped.WaitForCacheSync(ctx.Done())
}