2023-06-13 19:26:59 +00:00
// Copyright 2020-2023 the Pinniped contributors. All Rights Reserved.
2020-09-16 14:19:51 +00:00
// SPDX-License-Identifier: Apache-2.0
2020-08-07 21:49:04 +00:00
2020-09-21 18:16:14 +00:00
// Package controllermanager provides an entrypoint into running all of the controllers that run as
// a part of Pinniped.
2020-08-09 17:04:05 +00:00
package controllermanager
2020-08-07 21:49:04 +00:00
import (
"fmt"
"time"
k8sinformers "k8s.io/client-go/informers"
"k8s.io/client-go/kubernetes"
2021-12-10 22:22:36 +00:00
"k8s.io/utils/clock"
2020-08-07 21:49:04 +00:00
2021-02-16 19:00:08 +00:00
pinnipedclientset "go.pinniped.dev/generated/latest/client/concierge/clientset/versioned"
pinnipedinformers "go.pinniped.dev/generated/latest/client/concierge/informers/externalversions"
2021-02-10 16:12:03 +00:00
"go.pinniped.dev/internal/apiserviceref"
2021-02-12 01:22:47 +00:00
"go.pinniped.dev/internal/concierge/impersonator"
2020-10-15 19:40:56 +00:00
"go.pinniped.dev/internal/config/concierge"
2020-09-18 19:56:24 +00:00
"go.pinniped.dev/internal/controller/apicerts"
2020-10-30 19:02:21 +00:00
"go.pinniped.dev/internal/controller/authenticator/authncache"
2020-12-08 01:39:51 +00:00
"go.pinniped.dev/internal/controller/authenticator/cachecleaner"
"go.pinniped.dev/internal/controller/authenticator/jwtcachefiller"
2020-10-30 19:02:21 +00:00
"go.pinniped.dev/internal/controller/authenticator/webhookcachefiller"
2021-02-12 01:22:47 +00:00
"go.pinniped.dev/internal/controller/impersonatorconfig"
2020-09-21 18:16:14 +00:00
"go.pinniped.dev/internal/controller/kubecertagent"
2023-10-20 20:28:53 +00:00
"go.pinniped.dev/internal/controller/secrets"
2021-08-29 00:38:50 +00:00
"go.pinniped.dev/internal/controllerinit"
2020-09-18 19:56:24 +00:00
"go.pinniped.dev/internal/controllerlib"
2021-01-05 22:07:33 +00:00
"go.pinniped.dev/internal/deploymentref"
"go.pinniped.dev/internal/downward"
2020-09-23 12:26:59 +00:00
"go.pinniped.dev/internal/dynamiccert"
2021-01-13 01:27:41 +00:00
"go.pinniped.dev/internal/groupsuffix"
2021-01-05 22:07:33 +00:00
"go.pinniped.dev/internal/kubeclient"
2021-08-18 04:14:38 +00:00
"go.pinniped.dev/internal/leaderelection"
2022-04-16 02:43:53 +00:00
"go.pinniped.dev/internal/plog"
2023-10-20 20:28:53 +00:00
"go.pinniped.dev/internal/tokenclient"
2020-08-07 21:49:04 +00:00
)
const (
singletonWorker = 1
defaultResyncInterval = 3 * time . Minute
)
2020-09-21 18:16:14 +00:00
// Config holds all the input parameters to the set of controllers run as a part of Pinniped.
//
// It is used to inject parameters into PrepareControllers.
type Config struct {
2021-01-05 22:07:33 +00:00
// ServerInstallationInfo provides the name of the pod in which Pinniped is running and the namespace in which Pinniped is deployed.
ServerInstallationInfo * downward . PodInfo
2020-09-21 18:16:14 +00:00
2021-01-19 15:52:12 +00:00
// APIGroupSuffix is the suffix of the Pinniped API that should be targeted by these controllers.
APIGroupSuffix string
2020-09-21 18:16:14 +00:00
// NamesConfig comes from the Pinniped config API (see api.Config). It specifies how Kubernetes
// objects should be named.
2020-10-15 19:40:56 +00:00
NamesConfig * concierge . NamesConfigSpec
2020-09-21 18:16:14 +00:00
2020-09-24 00:30:22 +00:00
// KubeCertAgentConfig comes from the Pinniped config API (see api.Config). It configures how
// the kubecertagent package's controllers should manage the agent pods.
2020-10-15 19:40:56 +00:00
KubeCertAgentConfig * concierge . KubeCertAgentSpec
2020-09-24 00:30:22 +00:00
2021-11-17 21:27:59 +00:00
// ImpersonationProxyServerPort decides which port the impersonation proxy should bind.
ImpersonationProxyServerPort int
2020-09-21 18:16:14 +00:00
// DiscoveryURLOverride allows a caller to inject a hardcoded discovery URL into Pinniped
// discovery document.
DiscoveryURLOverride * string
2020-09-23 13:53:21 +00:00
// DynamicServingCertProvider provides a setter and a getter to the Pinniped API's serving cert.
2021-03-15 16:24:07 +00:00
DynamicServingCertProvider dynamiccert . Private
2021-03-12 00:27:16 +00:00
// DynamicSigningCertProvider provides a setter and a getter to the Pinniped API's
2020-09-23 15:01:41 +00:00
// signing cert, i.e., the cert that it uses to sign certs for Pinniped clients wishing to login.
2021-03-12 00:27:16 +00:00
// This is filled with the Kube API server's signing cert by a controller, if it can be found.
2021-03-15 16:24:07 +00:00
DynamicSigningCertProvider dynamiccert . Private
2021-03-12 00:27:16 +00:00
// ImpersonationSigningCertProvider provides a setter and a getter to the CA cert that should be
// used to sign client certs for authentication to the impersonation proxy. This CA is used by
// the TokenCredentialRequest to sign certs and by the impersonation proxy to check certs.
// When the impersonation proxy is not running, the getter will return nil cert and nil key.
// (Note that the impersonation proxy also accepts client certs signed by the Kube API server's cert.)
2021-03-10 18:30:06 +00:00
ImpersonationSigningCertProvider dynamiccert . Provider
2020-09-21 18:16:14 +00:00
2023-10-20 20:28:53 +00:00
// ImpersonationProxyTokenCache holds short-lived tokens for the impersonation proxy service account.
ImpersonationProxyTokenCache tokenclient . ExpiringSingletonTokenCacheGet
2020-09-21 18:16:14 +00:00
// ServingCertDuration is the validity period, in seconds, of the API serving certificate.
ServingCertDuration time . Duration
2021-03-12 00:27:16 +00:00
2020-09-21 18:16:14 +00:00
// ServingCertRenewBefore is the period of time, in seconds, that pinniped will wait before
// rotating the serving certificate. This period of time starts upon issuance of the serving
// certificate.
ServingCertRenewBefore time . Duration
2020-10-30 19:02:21 +00:00
// AuthenticatorCache is a cache of authenticators shared amongst various authenticated-related controllers.
AuthenticatorCache * authncache . Cache
2020-10-15 17:14:23 +00:00
// Labels are labels that should be added to any resources created by the controllers.
Labels map [ string ] string
2020-09-21 18:16:14 +00:00
}
2021-11-17 21:27:59 +00:00
// PrepareControllers prepares the controllers and their informers and returns a function that will start them when called.
2022-08-24 21:45:55 +00:00
func PrepareControllers ( c * Config ) ( controllerinit . RunnerBuilder , error ) { //nolint:funlen // Eh, fair, it is a really long function...but it is wiring the world...so...
2021-02-19 18:21:10 +00:00
loginConciergeGroupData , identityConciergeGroupData := groupsuffix . ConciergeAggregatedGroups ( c . APIGroupSuffix )
2021-02-10 16:12:03 +00:00
2021-12-15 20:48:55 +00:00
dref , deployment , _ , err := deploymentref . New ( c . ServerInstallationInfo )
2020-09-23 15:01:41 +00:00
if err != nil {
2021-01-05 22:07:33 +00:00
return nil , fmt . Errorf ( "cannot create deployment ref: %w" , err )
2020-09-23 15:01:41 +00:00
}
2021-01-05 22:07:33 +00:00
2021-02-19 18:21:10 +00:00
apiServiceRef , err := apiserviceref . New ( loginConciergeGroupData . APIServiceName ( ) )
2021-02-10 16:12:03 +00:00
if err != nil {
return nil , fmt . Errorf ( "cannot create API service ref: %w" , err )
}
2021-08-18 04:14:38 +00:00
client , leaderElector , err := leaderelection . New (
c . ServerInstallationInfo ,
deployment ,
2021-02-10 16:12:03 +00:00
dref , // first try to use the deployment as an owner ref (for namespace scoped resources)
apiServiceRef , // fallback to our API service (for everything else we create)
2021-01-13 01:27:41 +00:00
kubeclient . WithMiddleware ( groupsuffix . New ( c . APIGroupSuffix ) ) ,
)
2020-08-07 21:49:04 +00:00
if err != nil {
return nil , fmt . Errorf ( "could not create clients for the controllers: %w" , err )
}
2020-08-09 17:04:05 +00:00
// Create informers. Don't forget to make sure they get started in the function returned below.
2021-01-05 22:07:33 +00:00
informers := createInformers ( c . ServerInstallationInfo . Namespace , client . Kubernetes , client . PinnipedConcierge )
2020-08-07 21:49:04 +00:00
2021-04-20 19:55:28 +00:00
agentConfig := kubecertagent . AgentConfig {
2021-01-05 22:07:33 +00:00
Namespace : c . ServerInstallationInfo . Namespace ,
2021-05-03 21:31:48 +00:00
ServiceAccountName : c . NamesConfig . AgentServiceAccount ,
2020-09-24 19:52:05 +00:00
ContainerImage : * c . KubeCertAgentConfig . Image ,
2021-04-20 19:55:28 +00:00
NamePrefix : * c . KubeCertAgentConfig . NamePrefix ,
2020-09-24 19:52:05 +00:00
ContainerImagePullSecrets : c . KubeCertAgentConfig . ImagePullSecrets ,
2021-04-20 19:55:28 +00:00
Labels : c . Labels ,
CredentialIssuerName : c . NamesConfig . CredentialIssuer ,
DiscoveryURLOverride : c . DiscoveryURLOverride ,
2020-09-24 00:30:22 +00:00
}
2020-08-07 21:49:04 +00:00
// Create controller manager.
2020-08-28 15:59:09 +00:00
controllerManager := controllerlib .
2020-08-07 21:49:04 +00:00
NewManager ( ) .
2020-09-24 00:30:22 +00:00
// API certs controllers are responsible for managing the TLS certificates used to serve Pinniped's API.
2020-08-09 17:04:05 +00:00
WithController (
apicerts . NewCertsManagerController (
2021-01-05 22:07:33 +00:00
c . ServerInstallationInfo . Namespace ,
2020-09-21 18:16:14 +00:00
c . NamesConfig . ServingCertificateSecret ,
2020-10-15 17:14:23 +00:00
c . Labels ,
2021-01-05 22:07:33 +00:00
client . Kubernetes ,
2020-09-21 18:16:14 +00:00
informers . installationNamespaceK8s . Core ( ) . V1 ( ) . Secrets ( ) ,
2020-08-28 15:59:09 +00:00
controllerlib . WithInformer ,
controllerlib . WithInitialEvent ,
2020-09-21 18:16:14 +00:00
c . ServingCertDuration ,
2021-10-06 15:46:54 +00:00
"Pinniped Aggregation CA" ,
2020-09-21 18:16:14 +00:00
c . NamesConfig . APIService ,
2020-09-08 23:36:49 +00:00
) ,
singletonWorker ,
) .
2020-09-16 20:00:03 +00:00
WithController (
apicerts . NewAPIServiceUpdaterController (
2021-01-05 22:07:33 +00:00
c . ServerInstallationInfo . Namespace ,
2020-09-21 18:16:14 +00:00
c . NamesConfig . ServingCertificateSecret ,
2021-02-19 18:21:10 +00:00
loginConciergeGroupData . APIServiceName ( ) ,
client . Aggregation ,
informers . installationNamespaceK8s . Core ( ) . V1 ( ) . Secrets ( ) ,
controllerlib . WithInformer ,
) ,
singletonWorker ,
) .
WithController (
apicerts . NewAPIServiceUpdaterController (
c . ServerInstallationInfo . Namespace ,
c . NamesConfig . ServingCertificateSecret ,
identityConciergeGroupData . APIServiceName ( ) ,
2021-01-05 22:07:33 +00:00
client . Aggregation ,
2020-09-21 18:16:14 +00:00
informers . installationNamespaceK8s . Core ( ) . V1 ( ) . Secrets ( ) ,
2020-09-16 20:00:03 +00:00
controllerlib . WithInformer ,
) ,
singletonWorker ,
) .
2020-08-09 17:04:05 +00:00
WithController (
apicerts . NewCertsObserverController (
2021-01-05 22:07:33 +00:00
c . ServerInstallationInfo . Namespace ,
2020-09-21 18:16:14 +00:00
c . NamesConfig . ServingCertificateSecret ,
2020-09-23 13:53:21 +00:00
c . DynamicServingCertProvider ,
2020-09-21 18:16:14 +00:00
informers . installationNamespaceK8s . Core ( ) . V1 ( ) . Secrets ( ) ,
2020-08-28 15:59:09 +00:00
controllerlib . WithInformer ,
2020-08-07 21:49:04 +00:00
) ,
singletonWorker ,
2020-08-19 20:15:45 +00:00
) .
WithController (
apicerts . NewCertsExpirerController (
2021-01-05 22:07:33 +00:00
c . ServerInstallationInfo . Namespace ,
2020-09-21 18:16:14 +00:00
c . NamesConfig . ServingCertificateSecret ,
2021-01-05 22:07:33 +00:00
client . Kubernetes ,
2020-09-21 18:16:14 +00:00
informers . installationNamespaceK8s . Core ( ) . V1 ( ) . Secrets ( ) ,
2020-08-28 15:59:09 +00:00
controllerlib . WithInformer ,
2020-09-21 18:16:14 +00:00
c . ServingCertRenewBefore ,
2021-03-10 18:30:06 +00:00
apicerts . TLSCertificateChainSecretKey ,
2022-04-16 02:43:53 +00:00
plog . New ( ) ,
2020-08-19 20:15:45 +00:00
) ,
singletonWorker ,
2020-09-14 15:47:16 +00:00
) .
2021-04-20 19:55:28 +00:00
// The kube-cert-agent controller is responsible for finding the cluster's signing keys and keeping them
2020-09-24 00:30:22 +00:00
// up to date in memory, as well as reporting status on this cluster integration strategy.
2020-09-14 15:47:16 +00:00
WithController (
2021-04-20 19:55:28 +00:00
kubecertagent . NewAgentController (
agentConfig ,
client ,
2020-09-24 00:30:22 +00:00
informers . kubeSystemNamespaceK8s . Core ( ) . V1 ( ) . Pods ( ) ,
2021-04-20 19:55:28 +00:00
informers . installationNamespaceK8s . Apps ( ) . V1 ( ) . Deployments ( ) ,
2020-09-21 23:37:22 +00:00
informers . installationNamespaceK8s . Core ( ) . V1 ( ) . Pods ( ) ,
2021-03-02 18:55:24 +00:00
informers . kubePublicNamespaceK8s . Core ( ) . V1 ( ) . ConfigMaps ( ) ,
2021-05-19 21:53:00 +00:00
informers . pinniped . Config ( ) . V1alpha1 ( ) . CredentialIssuers ( ) ,
2021-04-20 19:55:28 +00:00
c . DynamicSigningCertProvider ,
2020-09-21 18:16:14 +00:00
) ,
singletonWorker ,
) .
2021-04-20 19:56:43 +00:00
// The kube-cert-agent legacy pod cleaner controller is responsible for cleaning up pods that were deployed by
// versions of Pinniped prior to v0.7.0. If we stop supporting upgrades from v0.7.0, we can safely remove this.
WithController (
kubecertagent . NewLegacyPodCleanerController (
agentConfig ,
client ,
informers . installationNamespaceK8s . Core ( ) . V1 ( ) . Pods ( ) ,
2023-06-13 19:26:59 +00:00
plog . Logr ( ) , //nolint:staticcheck // old controller with lots of log statements
2021-04-20 19:56:43 +00:00
) ,
singletonWorker ,
) .
2020-10-30 19:02:21 +00:00
// The cache filler/cleaner controllers are responsible for keep an in-memory representation of active
// authenticators up to date.
2020-09-21 18:16:14 +00:00
WithController (
2020-09-24 00:30:22 +00:00
webhookcachefiller . New (
2020-10-30 19:02:21 +00:00
c . AuthenticatorCache ,
2021-02-09 18:59:32 +00:00
informers . pinniped . Authentication ( ) . V1alpha1 ( ) . WebhookAuthenticators ( ) ,
2023-06-13 19:26:59 +00:00
plog . Logr ( ) , //nolint:staticcheck // old controller with lots of log statements
2020-09-21 18:16:14 +00:00
) ,
singletonWorker ,
2020-09-23 15:01:41 +00:00
) .
WithController (
2020-12-08 01:39:51 +00:00
jwtcachefiller . New (
c . AuthenticatorCache ,
2021-02-09 18:59:32 +00:00
informers . pinniped . Authentication ( ) . V1alpha1 ( ) . JWTAuthenticators ( ) ,
2023-06-13 19:26:59 +00:00
plog . Logr ( ) , //nolint:staticcheck // old controller with lots of log statements
2020-12-08 01:39:51 +00:00
) ,
singletonWorker ,
) .
WithController (
cachecleaner . New (
2020-10-30 19:02:21 +00:00
c . AuthenticatorCache ,
2021-02-09 18:59:32 +00:00
informers . pinniped . Authentication ( ) . V1alpha1 ( ) . WebhookAuthenticators ( ) ,
informers . pinniped . Authentication ( ) . V1alpha1 ( ) . JWTAuthenticators ( ) ,
2023-06-13 19:26:59 +00:00
plog . Logr ( ) , //nolint:staticcheck // old controller with lots of log statements
2020-09-23 15:01:41 +00:00
) ,
singletonWorker ,
2021-02-12 01:22:47 +00:00
) .
2021-03-02 22:48:58 +00:00
// The impersonator configuration controller dynamically configures the impersonation proxy feature.
2021-02-12 01:22:47 +00:00
WithController (
impersonatorconfig . NewImpersonatorConfigController (
c . ServerInstallationInfo . Namespace ,
2021-03-02 22:48:58 +00:00
c . NamesConfig . CredentialIssuer ,
2021-02-12 01:22:47 +00:00
client . Kubernetes ,
2021-03-02 22:48:58 +00:00
client . PinnipedConcierge ,
2021-05-17 22:08:05 +00:00
informers . pinniped . Config ( ) . V1alpha1 ( ) . CredentialIssuers ( ) ,
2021-02-18 01:22:13 +00:00
informers . installationNamespaceK8s . Core ( ) . V1 ( ) . Services ( ) ,
2021-02-24 18:56:24 +00:00
informers . installationNamespaceK8s . Core ( ) . V1 ( ) . Secrets ( ) ,
2021-02-12 01:22:47 +00:00
controllerlib . WithInformer ,
2021-11-17 21:27:59 +00:00
c . ImpersonationProxyServerPort ,
2021-03-02 17:31:24 +00:00
c . NamesConfig . ImpersonationLoadBalancerService ,
2021-05-20 21:11:35 +00:00
c . NamesConfig . ImpersonationClusterIPService ,
2021-03-02 17:31:24 +00:00
c . NamesConfig . ImpersonationTLSCertificateSecret ,
c . NamesConfig . ImpersonationCACertificateSecret ,
2021-02-16 23:57:02 +00:00
c . Labels ,
2021-03-02 22:48:58 +00:00
clock . RealClock { } ,
2021-03-10 18:30:06 +00:00
impersonator . New ,
c . NamesConfig . ImpersonationSignerSecret ,
c . ImpersonationSigningCertProvider ,
2023-06-13 19:26:59 +00:00
plog . Logr ( ) , //nolint:staticcheck // old controller with lots of log statements
2023-10-20 20:28:53 +00:00
c . ImpersonationProxyTokenCache ,
2021-03-10 18:30:06 +00:00
) ,
singletonWorker ,
) .
WithController (
apicerts . NewCertsManagerController (
c . ServerInstallationInfo . Namespace ,
c . NamesConfig . ImpersonationSignerSecret ,
c . Labels ,
client . Kubernetes ,
informers . installationNamespaceK8s . Core ( ) . V1 ( ) . Secrets ( ) ,
controllerlib . WithInformer ,
controllerlib . WithInitialEvent ,
365 * 24 * time . Hour , // 1 year hard coded value
2021-10-06 15:46:54 +00:00
"Pinniped Impersonation Proxy Signer CA" ,
2021-03-10 18:30:06 +00:00
"" , // optional, means do not give me a serving cert
) ,
singletonWorker ,
) .
WithController (
apicerts . NewCertsExpirerController (
c . ServerInstallationInfo . Namespace ,
c . NamesConfig . ImpersonationSignerSecret ,
client . Kubernetes ,
informers . installationNamespaceK8s . Core ( ) . V1 ( ) . Secrets ( ) ,
controllerlib . WithInformer ,
2021-10-06 15:46:54 +00:00
365 * 24 * time . Hour - time . Hour , // 1 year minus 1 hour hard coded value (i.e. wait until the last moment to break the signer)
2021-03-10 18:30:06 +00:00
apicerts . CACertificateSecretKey ,
2022-04-16 02:43:53 +00:00
plog . New ( ) ,
2021-02-12 01:22:47 +00:00
) ,
singletonWorker ,
2023-10-20 20:28:53 +00:00
) .
WithController (
secrets . NewServiceAccountTokenCleanupController (
c . ServerInstallationInfo . Namespace ,
c . NamesConfig . ImpersonationProxyLegacySecret ,
client . Kubernetes ,
informers . installationNamespaceK8s . Core ( ) . V1 ( ) . Secrets ( ) ,
controllerlib . WithInformer ,
plog . New ( ) ,
) ,
singletonWorker ,
2020-08-07 21:49:04 +00:00
)
2021-08-29 00:38:50 +00:00
return controllerinit . Prepare ( controllerManager . Start , leaderElector ,
informers . kubePublicNamespaceK8s ,
informers . kubeSystemNamespaceK8s ,
informers . installationNamespaceK8s ,
informers . pinniped ,
) , nil
2020-08-07 21:49:04 +00:00
}
2020-09-21 18:16:14 +00:00
type informers struct {
2021-02-09 18:59:32 +00:00
kubePublicNamespaceK8s k8sinformers . SharedInformerFactory
kubeSystemNamespaceK8s k8sinformers . SharedInformerFactory
installationNamespaceK8s k8sinformers . SharedInformerFactory
pinniped pinnipedinformers . SharedInformerFactory
2020-09-21 18:16:14 +00:00
}
2020-08-07 21:49:04 +00:00
// Create the informers that will be used by the controllers.
func createInformers (
serverInstallationNamespace string ,
2021-01-05 22:07:33 +00:00
k8sClient kubernetes . Interface ,
pinnipedClient pinnipedclientset . Interface ,
2020-09-21 18:16:14 +00:00
) * informers {
return & informers {
kubePublicNamespaceK8s : k8sinformers . NewSharedInformerFactoryWithOptions (
k8sClient ,
defaultResyncInterval ,
2021-03-02 18:55:24 +00:00
k8sinformers . WithNamespace ( kubecertagent . ClusterInfoNamespace ) ,
2020-09-21 18:16:14 +00:00
) ,
kubeSystemNamespaceK8s : k8sinformers . NewSharedInformerFactoryWithOptions (
k8sClient ,
defaultResyncInterval ,
k8sinformers . WithNamespace ( kubecertagent . ControllerManagerNamespace ) ,
) ,
installationNamespaceK8s : k8sinformers . NewSharedInformerFactoryWithOptions (
k8sClient ,
defaultResyncInterval ,
k8sinformers . WithNamespace ( serverInstallationNamespace ) ,
) ,
2021-02-09 18:59:32 +00:00
pinniped : pinnipedinformers . NewSharedInformerFactoryWithOptions (
2020-09-21 18:16:14 +00:00
pinnipedClient ,
defaultResyncInterval ,
) ,
}
}