657488fe90
Previously, our controllers would automatically create a CredentialIssuer with a singleton name. The helpers we had for this also used "raw" client access and did not take advantage of the informer cache pattern. With this change, the CredentialIssuer is always created at install time in the ytt YAML. The controllers now only update the existing CredentialIssuer status, and they do so using the informer cache as much as possible. This change is targeted at only the kubecertagent controller to start. The impersonatorconfig controller will be updated in a following PR along with other changes. Signed-off-by: Matt Moyer <moyerm@vmware.com>
357 lines
13 KiB
Go
357 lines
13 KiB
Go
// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved.
|
|
// SPDX-License-Identifier: Apache-2.0
|
|
|
|
// Package controllermanager provides an entrypoint into running all of the controllers that run as
|
|
// a part of Pinniped.
|
|
package controllermanager
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"time"
|
|
|
|
"k8s.io/apimachinery/pkg/util/clock"
|
|
k8sinformers "k8s.io/client-go/informers"
|
|
"k8s.io/client-go/kubernetes"
|
|
"k8s.io/klog/v2/klogr"
|
|
|
|
pinnipedclientset "go.pinniped.dev/generated/latest/client/concierge/clientset/versioned"
|
|
pinnipedinformers "go.pinniped.dev/generated/latest/client/concierge/informers/externalversions"
|
|
"go.pinniped.dev/internal/apiserviceref"
|
|
"go.pinniped.dev/internal/concierge/impersonator"
|
|
"go.pinniped.dev/internal/config/concierge"
|
|
"go.pinniped.dev/internal/controller/apicerts"
|
|
"go.pinniped.dev/internal/controller/authenticator/authncache"
|
|
"go.pinniped.dev/internal/controller/authenticator/cachecleaner"
|
|
"go.pinniped.dev/internal/controller/authenticator/jwtcachefiller"
|
|
"go.pinniped.dev/internal/controller/authenticator/webhookcachefiller"
|
|
"go.pinniped.dev/internal/controller/impersonatorconfig"
|
|
"go.pinniped.dev/internal/controller/kubecertagent"
|
|
"go.pinniped.dev/internal/controllerlib"
|
|
"go.pinniped.dev/internal/deploymentref"
|
|
"go.pinniped.dev/internal/downward"
|
|
"go.pinniped.dev/internal/dynamiccert"
|
|
"go.pinniped.dev/internal/groupsuffix"
|
|
"go.pinniped.dev/internal/kubeclient"
|
|
)
|
|
|
|
const (
|
|
singletonWorker = 1
|
|
defaultResyncInterval = 3 * time.Minute
|
|
)
|
|
|
|
// Config holds all the input parameters to the set of controllers run as a part of Pinniped.
|
|
//
|
|
// It is used to inject parameters into PrepareControllers.
|
|
type Config struct {
|
|
// ServerInstallationInfo provides the name of the pod in which Pinniped is running and the namespace in which Pinniped is deployed.
|
|
ServerInstallationInfo *downward.PodInfo
|
|
|
|
// APIGroupSuffix is the suffix of the Pinniped API that should be targeted by these controllers.
|
|
APIGroupSuffix string
|
|
|
|
// NamesConfig comes from the Pinniped config API (see api.Config). It specifies how Kubernetes
|
|
// objects should be named.
|
|
NamesConfig *concierge.NamesConfigSpec
|
|
|
|
// KubeCertAgentConfig comes from the Pinniped config API (see api.Config). It configures how
|
|
// the kubecertagent package's controllers should manage the agent pods.
|
|
KubeCertAgentConfig *concierge.KubeCertAgentSpec
|
|
|
|
// DiscoveryURLOverride allows a caller to inject a hardcoded discovery URL into Pinniped
|
|
// discovery document.
|
|
DiscoveryURLOverride *string
|
|
|
|
// DynamicServingCertProvider provides a setter and a getter to the Pinniped API's serving cert.
|
|
DynamicServingCertProvider dynamiccert.Private
|
|
|
|
// DynamicSigningCertProvider provides a setter and a getter to the Pinniped API's
|
|
// signing cert, i.e., the cert that it uses to sign certs for Pinniped clients wishing to login.
|
|
// This is filled with the Kube API server's signing cert by a controller, if it can be found.
|
|
DynamicSigningCertProvider dynamiccert.Private
|
|
|
|
// ImpersonationSigningCertProvider provides a setter and a getter to the CA cert that should be
|
|
// used to sign client certs for authentication to the impersonation proxy. This CA is used by
|
|
// the TokenCredentialRequest to sign certs and by the impersonation proxy to check certs.
|
|
// When the impersonation proxy is not running, the getter will return nil cert and nil key.
|
|
// (Note that the impersonation proxy also accepts client certs signed by the Kube API server's cert.)
|
|
ImpersonationSigningCertProvider dynamiccert.Provider
|
|
|
|
// ServingCertDuration is the validity period, in seconds, of the API serving certificate.
|
|
ServingCertDuration time.Duration
|
|
|
|
// ServingCertRenewBefore is the period of time, in seconds, that pinniped will wait before
|
|
// rotating the serving certificate. This period of time starts upon issuance of the serving
|
|
// certificate.
|
|
ServingCertRenewBefore time.Duration
|
|
|
|
// AuthenticatorCache is a cache of authenticators shared amongst various authenticated-related controllers.
|
|
AuthenticatorCache *authncache.Cache
|
|
|
|
// Labels are labels that should be added to any resources created by the controllers.
|
|
Labels map[string]string
|
|
}
|
|
|
|
// Prepare the controllers and their informers and return a function that will start them when called.
|
|
//nolint:funlen // Eh, fair, it is a really long function...but it is wiring the world...so...
|
|
func PrepareControllers(c *Config) (func(ctx context.Context), error) {
|
|
loginConciergeGroupData, identityConciergeGroupData := groupsuffix.ConciergeAggregatedGroups(c.APIGroupSuffix)
|
|
|
|
dref, _, err := deploymentref.New(c.ServerInstallationInfo)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("cannot create deployment ref: %w", err)
|
|
}
|
|
|
|
apiServiceRef, err := apiserviceref.New(loginConciergeGroupData.APIServiceName())
|
|
if err != nil {
|
|
return nil, fmt.Errorf("cannot create API service ref: %w", err)
|
|
}
|
|
|
|
client, err := kubeclient.New(
|
|
dref, // first try to use the deployment as an owner ref (for namespace scoped resources)
|
|
apiServiceRef, // fallback to our API service (for everything else we create)
|
|
kubeclient.WithMiddleware(groupsuffix.New(c.APIGroupSuffix)),
|
|
)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("could not create clients for the controllers: %w", err)
|
|
}
|
|
|
|
// Create informers. Don't forget to make sure they get started in the function returned below.
|
|
informers := createInformers(c.ServerInstallationInfo.Namespace, client.Kubernetes, client.PinnipedConcierge)
|
|
|
|
agentConfig := kubecertagent.AgentConfig{
|
|
Namespace: c.ServerInstallationInfo.Namespace,
|
|
ServiceAccountName: c.NamesConfig.AgentServiceAccount,
|
|
ContainerImage: *c.KubeCertAgentConfig.Image,
|
|
NamePrefix: *c.KubeCertAgentConfig.NamePrefix,
|
|
ContainerImagePullSecrets: c.KubeCertAgentConfig.ImagePullSecrets,
|
|
Labels: c.Labels,
|
|
CredentialIssuerName: c.NamesConfig.CredentialIssuer,
|
|
DiscoveryURLOverride: c.DiscoveryURLOverride,
|
|
}
|
|
|
|
// Create controller manager.
|
|
controllerManager := controllerlib.
|
|
NewManager().
|
|
|
|
// API certs controllers are responsible for managing the TLS certificates used to serve Pinniped's API.
|
|
WithController(
|
|
apicerts.NewCertsManagerController(
|
|
c.ServerInstallationInfo.Namespace,
|
|
c.NamesConfig.ServingCertificateSecret,
|
|
c.Labels,
|
|
client.Kubernetes,
|
|
informers.installationNamespaceK8s.Core().V1().Secrets(),
|
|
controllerlib.WithInformer,
|
|
controllerlib.WithInitialEvent,
|
|
c.ServingCertDuration,
|
|
"Pinniped CA",
|
|
c.NamesConfig.APIService,
|
|
),
|
|
singletonWorker,
|
|
).
|
|
WithController(
|
|
apicerts.NewAPIServiceUpdaterController(
|
|
c.ServerInstallationInfo.Namespace,
|
|
c.NamesConfig.ServingCertificateSecret,
|
|
loginConciergeGroupData.APIServiceName(),
|
|
client.Aggregation,
|
|
informers.installationNamespaceK8s.Core().V1().Secrets(),
|
|
controllerlib.WithInformer,
|
|
),
|
|
singletonWorker,
|
|
).
|
|
WithController(
|
|
apicerts.NewAPIServiceUpdaterController(
|
|
c.ServerInstallationInfo.Namespace,
|
|
c.NamesConfig.ServingCertificateSecret,
|
|
identityConciergeGroupData.APIServiceName(),
|
|
client.Aggregation,
|
|
informers.installationNamespaceK8s.Core().V1().Secrets(),
|
|
controllerlib.WithInformer,
|
|
),
|
|
singletonWorker,
|
|
).
|
|
WithController(
|
|
apicerts.NewCertsObserverController(
|
|
c.ServerInstallationInfo.Namespace,
|
|
c.NamesConfig.ServingCertificateSecret,
|
|
c.DynamicServingCertProvider,
|
|
informers.installationNamespaceK8s.Core().V1().Secrets(),
|
|
controllerlib.WithInformer,
|
|
),
|
|
singletonWorker,
|
|
).
|
|
WithController(
|
|
apicerts.NewCertsExpirerController(
|
|
c.ServerInstallationInfo.Namespace,
|
|
c.NamesConfig.ServingCertificateSecret,
|
|
client.Kubernetes,
|
|
informers.installationNamespaceK8s.Core().V1().Secrets(),
|
|
controllerlib.WithInformer,
|
|
c.ServingCertRenewBefore,
|
|
apicerts.TLSCertificateChainSecretKey,
|
|
),
|
|
singletonWorker,
|
|
).
|
|
// The kube-cert-agent controller is responsible for finding the cluster's signing keys and keeping them
|
|
// up to date in memory, as well as reporting status on this cluster integration strategy.
|
|
WithController(
|
|
kubecertagent.NewAgentController(
|
|
agentConfig,
|
|
client,
|
|
informers.kubeSystemNamespaceK8s.Core().V1().Pods(),
|
|
informers.installationNamespaceK8s.Apps().V1().Deployments(),
|
|
informers.installationNamespaceK8s.Core().V1().Pods(),
|
|
informers.kubePublicNamespaceK8s.Core().V1().ConfigMaps(),
|
|
informers.pinniped.Config().V1alpha1().CredentialIssuers(),
|
|
c.DynamicSigningCertProvider,
|
|
),
|
|
singletonWorker,
|
|
).
|
|
// The kube-cert-agent legacy pod cleaner controller is responsible for cleaning up pods that were deployed by
|
|
// versions of Pinniped prior to v0.7.0. If we stop supporting upgrades from v0.7.0, we can safely remove this.
|
|
WithController(
|
|
kubecertagent.NewLegacyPodCleanerController(
|
|
agentConfig,
|
|
client,
|
|
informers.installationNamespaceK8s.Core().V1().Pods(),
|
|
klogr.New(),
|
|
),
|
|
singletonWorker,
|
|
).
|
|
// The cache filler/cleaner controllers are responsible for keep an in-memory representation of active
|
|
// authenticators up to date.
|
|
WithController(
|
|
webhookcachefiller.New(
|
|
c.AuthenticatorCache,
|
|
informers.pinniped.Authentication().V1alpha1().WebhookAuthenticators(),
|
|
klogr.New(),
|
|
),
|
|
singletonWorker,
|
|
).
|
|
WithController(
|
|
jwtcachefiller.New(
|
|
c.AuthenticatorCache,
|
|
informers.pinniped.Authentication().V1alpha1().JWTAuthenticators(),
|
|
klogr.New(),
|
|
),
|
|
singletonWorker,
|
|
).
|
|
WithController(
|
|
cachecleaner.New(
|
|
c.AuthenticatorCache,
|
|
informers.pinniped.Authentication().V1alpha1().WebhookAuthenticators(),
|
|
informers.pinniped.Authentication().V1alpha1().JWTAuthenticators(),
|
|
klogr.New(),
|
|
),
|
|
singletonWorker,
|
|
).
|
|
|
|
// The impersonator configuration controller dynamically configures the impersonation proxy feature.
|
|
WithController(
|
|
impersonatorconfig.NewImpersonatorConfigController(
|
|
c.ServerInstallationInfo.Namespace,
|
|
c.NamesConfig.ImpersonationConfigMap,
|
|
c.NamesConfig.CredentialIssuer,
|
|
client.Kubernetes,
|
|
client.PinnipedConcierge,
|
|
informers.installationNamespaceK8s.Core().V1().ConfigMaps(),
|
|
informers.installationNamespaceK8s.Core().V1().Services(),
|
|
informers.installationNamespaceK8s.Core().V1().Secrets(),
|
|
controllerlib.WithInformer,
|
|
controllerlib.WithInitialEvent,
|
|
c.NamesConfig.ImpersonationLoadBalancerService,
|
|
c.NamesConfig.ImpersonationTLSCertificateSecret,
|
|
c.NamesConfig.ImpersonationCACertificateSecret,
|
|
c.Labels,
|
|
clock.RealClock{},
|
|
impersonator.New,
|
|
c.NamesConfig.ImpersonationSignerSecret,
|
|
c.ImpersonationSigningCertProvider,
|
|
),
|
|
singletonWorker,
|
|
).
|
|
WithController(
|
|
apicerts.NewCertsManagerController(
|
|
c.ServerInstallationInfo.Namespace,
|
|
c.NamesConfig.ImpersonationSignerSecret,
|
|
c.Labels,
|
|
client.Kubernetes,
|
|
informers.installationNamespaceK8s.Core().V1().Secrets(),
|
|
controllerlib.WithInformer,
|
|
controllerlib.WithInitialEvent,
|
|
365*24*time.Hour, // 1 year hard coded value
|
|
"Pinniped Impersonation Proxy CA",
|
|
"", // optional, means do not give me a serving cert
|
|
),
|
|
singletonWorker,
|
|
).
|
|
WithController(
|
|
apicerts.NewCertsExpirerController(
|
|
c.ServerInstallationInfo.Namespace,
|
|
c.NamesConfig.ImpersonationSignerSecret,
|
|
client.Kubernetes,
|
|
informers.installationNamespaceK8s.Core().V1().Secrets(),
|
|
controllerlib.WithInformer,
|
|
c.ServingCertRenewBefore,
|
|
apicerts.CACertificateSecretKey,
|
|
),
|
|
singletonWorker,
|
|
)
|
|
|
|
// Return a function which starts the informers and controllers.
|
|
return func(ctx context.Context) {
|
|
informers.startAndWaitForSync(ctx)
|
|
go controllerManager.Start(ctx)
|
|
}, nil
|
|
}
|
|
|
|
type informers struct {
|
|
kubePublicNamespaceK8s k8sinformers.SharedInformerFactory
|
|
kubeSystemNamespaceK8s k8sinformers.SharedInformerFactory
|
|
installationNamespaceK8s k8sinformers.SharedInformerFactory
|
|
pinniped pinnipedinformers.SharedInformerFactory
|
|
}
|
|
|
|
// Create the informers that will be used by the controllers.
|
|
func createInformers(
|
|
serverInstallationNamespace string,
|
|
k8sClient kubernetes.Interface,
|
|
pinnipedClient pinnipedclientset.Interface,
|
|
) *informers {
|
|
return &informers{
|
|
kubePublicNamespaceK8s: k8sinformers.NewSharedInformerFactoryWithOptions(
|
|
k8sClient,
|
|
defaultResyncInterval,
|
|
k8sinformers.WithNamespace(kubecertagent.ClusterInfoNamespace),
|
|
),
|
|
kubeSystemNamespaceK8s: k8sinformers.NewSharedInformerFactoryWithOptions(
|
|
k8sClient,
|
|
defaultResyncInterval,
|
|
k8sinformers.WithNamespace(kubecertagent.ControllerManagerNamespace),
|
|
),
|
|
installationNamespaceK8s: k8sinformers.NewSharedInformerFactoryWithOptions(
|
|
k8sClient,
|
|
defaultResyncInterval,
|
|
k8sinformers.WithNamespace(serverInstallationNamespace),
|
|
),
|
|
pinniped: pinnipedinformers.NewSharedInformerFactoryWithOptions(
|
|
pinnipedClient,
|
|
defaultResyncInterval,
|
|
),
|
|
}
|
|
}
|
|
|
|
func (i *informers) startAndWaitForSync(ctx context.Context) {
|
|
i.kubePublicNamespaceK8s.Start(ctx.Done())
|
|
i.kubeSystemNamespaceK8s.Start(ctx.Done())
|
|
i.installationNamespaceK8s.Start(ctx.Done())
|
|
i.pinniped.Start(ctx.Done())
|
|
|
|
i.kubePublicNamespaceK8s.WaitForCacheSync(ctx.Done())
|
|
i.kubeSystemNamespaceK8s.WaitForCacheSync(ctx.Done())
|
|
i.installationNamespaceK8s.WaitForCacheSync(ctx.Done())
|
|
i.pinniped.WaitForCacheSync(ctx.Done())
|
|
}
|