From 86c3f89b2e13074ba88f0a02a275c58c222bd480 Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Sun, 9 Aug 2020 10:04:05 -0700 Subject: [PATCH] First draft of moving API server TLS cert generation to controllers - Refactors the existing cert generation code into controllers which read and write a Secret containing the certs - Does not add any new functionality yet, e.g. no new handling for cert expiration, and no leader election to allow for multiple servers running simultaneously - This commit also doesn't add new tests for the cert generation code, but it should be more unit testable now as controllers --- Dockerfile | 3 + README.md | 4 +- deploy/rbac.yaml | 17 +- internal/autoregistration/autoregistration.go | 6 +- internal/controller/apicerts/certs_manager.go | 164 ++++++++++++++++++ .../controller/apicerts/certs_observer.go | 69 ++++++++ .../controller/logindiscovery/publisher.go | 26 +-- internal/controller/utils.go | 31 ++++ .../prepare_controllers.go | 89 ++++++---- .../dynamic_tls_serving_cert_provider.go | 19 ++ internal/server/server.go | 81 +++------ 11 files changed, 391 insertions(+), 118 deletions(-) create mode 100644 internal/controller/apicerts/certs_manager.go create mode 100644 internal/controller/apicerts/certs_observer.go create mode 100644 internal/controller/utils.go rename internal/{controller => controllermanager}/prepare_controllers.go (56%) create mode 100644 internal/provider/dynamic_tls_serving_cert_provider.go diff --git a/Dockerfile b/Dockerfile index f03e2545..00eaa5d4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,6 @@ +# Copyright 2020 VMware, Inc. +# SPDX-License-Identifier: Apache-2.0 + FROM golang:1.14-alpine as build-env # It is important that these ARG's are defined after the FROM statement diff --git a/README.md b/README.md index a84c0d2c..49b59c26 100644 --- a/README.md +++ b/README.md @@ -1 +1,3 @@ -# placeholder-name \ No newline at end of file +# placeholder-name + +Copyright 2020 VMware, Inc. diff --git a/deploy/rbac.yaml b/deploy/rbac.yaml index 98caf1c3..75b920d8 100644 --- a/deploy/rbac.yaml +++ b/deploy/rbac.yaml @@ -1,5 +1,6 @@ #@ load("@ytt:data", "data") +#! Give permission to various cluster-scoped objects --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole @@ -28,6 +29,8 @@ roleRef: kind: ClusterRole name: #@ data.values.app_name + "-aggregated-api-server-cluster-role" apiGroup: rbac.authorization.k8s.io + +#! Give permission to various objects within the app's own namespace --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role @@ -38,6 +41,9 @@ rules: - apiGroups: [""] resources: [services] verbs: [create, get, list, patch, update, watch] + - apiGroups: [""] + resources: [secrets] + verbs: [create, get, list, patch, update, watch] - apiGroups: [crds.placeholder.suzerain-io.github.io] resources: [logindiscoveryconfigs] verbs: [create, get, list, update, watch] @@ -55,6 +61,8 @@ roleRef: kind: Role name: #@ data.values.app_name + "-aggregated-api-server-role" apiGroup: rbac.authorization.k8s.io + +#! Allow both authenticated and unauthenticated LoginRequests (i.e. allow all requests) --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole @@ -70,7 +78,6 @@ apiVersion: rbac.authorization.k8s.io/v1 metadata: name: #@ data.values.app_name + "-loginrequests-cluster-role-binding" subjects: - #! both authenticated and unauthenticated requests (i.e. all requests) should be allowed - kind: Group name: system:authenticated apiGroup: rbac.authorization.k8s.io @@ -81,6 +88,8 @@ roleRef: kind: ClusterRole name: #@ data.values.app_name + "-loginrequests-cluster-role" apiGroup: rbac.authorization.k8s.io + +#! Give permissions for subjectaccessreviews, tokenreview that is needed by aggregated api servers --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 @@ -93,9 +102,10 @@ subjects: namespace: #@ data.values.namespace roleRef: kind: ClusterRole - #! give permissions for subjectaccessreviews, tokenreview that is needed by aggregated api servers name: system:auth-delegator apiGroup: rbac.authorization.k8s.io + +#! Give permissions for a special configmap of CA bundles that is needed by aggregated api servers --- kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 @@ -108,9 +118,10 @@ subjects: namespace: #@ data.values.namespace roleRef: kind: Role - #! give permissions for a special configmap of CA bundles that is needed by aggregated api servers name: extension-apiserver-authentication-reader apiGroup: rbac.authorization.k8s.io + +#! Give permission to list and watch ConfigMaps in kube-public --- kind: Role apiVersion: rbac.authorization.k8s.io/v1 diff --git a/internal/autoregistration/autoregistration.go b/internal/autoregistration/autoregistration.go index a729ed70..7033abe4 100644 --- a/internal/autoregistration/autoregistration.go +++ b/internal/autoregistration/autoregistration.go @@ -12,14 +12,14 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/util/retry" - aggregatationv1client "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset" + aggregatorclient "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset" placeholderv1alpha1 "github.com/suzerain-io/placeholder-name/kubernetes/1.19/api/apis/placeholder/v1alpha1" ) // UpdateAPIService updates the APIService's CA bundle. -func UpdateAPIService(ctx context.Context, aggregationV1 aggregatationv1client.Interface, aggregatedAPIServerCA []byte) error { - apiServices := aggregationV1.ApiregistrationV1().APIServices() +func UpdateAPIService(ctx context.Context, aggregatorClient aggregatorclient.Interface, aggregatedAPIServerCA []byte) error { + apiServices := aggregatorClient.ApiregistrationV1().APIServices() apiServiceName := placeholderv1alpha1.SchemeGroupVersion.Version + "." + placeholderv1alpha1.GroupName if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { diff --git a/internal/controller/apicerts/certs_manager.go b/internal/controller/apicerts/certs_manager.go new file mode 100644 index 00000000..a6efe81f --- /dev/null +++ b/internal/controller/apicerts/certs_manager.go @@ -0,0 +1,164 @@ +/* +Copyright 2020 VMware, Inc. +SPDX-License-Identifier: Apache-2.0 +*/ + +package apicerts + +import ( + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "time" + + corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + corev1informers "k8s.io/client-go/informers/core/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/klog/v2" + aggregatorclient "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset" + + "github.com/suzerain-io/controller-go" + "github.com/suzerain-io/placeholder-name/internal/autoregistration" + "github.com/suzerain-io/placeholder-name/internal/certauthority" + placeholdernamecontroller "github.com/suzerain-io/placeholder-name/internal/controller" + placeholderv1alpha1 "github.com/suzerain-io/placeholder-name/kubernetes/1.19/api/apis/placeholder/v1alpha1" +) + +const ( + //nolint: gosec + certsSecretName = "api-serving-cert" + caCertificateSecretKey = "caCertificate" + tlsPrivateKeySecretKey = "tlsPrivateKey" + tlsCertificateChainSecretKey = "tlsCertificateChain" +) + +type certsManagerController struct { + namespace string + apiServiceName string + k8sClient kubernetes.Interface + aggregatorClient *aggregatorclient.Clientset + secretInformer corev1informers.SecretInformer +} + +func NewCertsManagerController( + namespace string, + k8sClient kubernetes.Interface, + aggregationClient *aggregatorclient.Clientset, + secretInformer corev1informers.SecretInformer, + withInformer placeholdernamecontroller.WithInformerOptionFunc, +) controller.Controller { + apiServiceName := placeholderv1alpha1.SchemeGroupVersion.Version + "." + placeholderv1alpha1.GroupName + return controller.New( + controller.Config{ + Name: "certs-manager-controller", + Syncer: &certsManagerController{ + apiServiceName: apiServiceName, + namespace: namespace, + k8sClient: k8sClient, + aggregatorClient: aggregationClient, + secretInformer: secretInformer, + }, + }, + withInformer( + secretInformer, + placeholdernamecontroller.NameAndNamespaceExactMatchFilterFactory(certsSecretName, namespace), + controller.InformerOption{}, + ), + // Be sure to run once even if the Secret that the informer is watching doesn't exist. + controller.WithInitialEvent(controller.Key{ + Namespace: namespace, + Name: certsSecretName, + }), + ) +} + +func (c *certsManagerController) Sync(ctx controller.Context) error { + // Try to get the secret from the informer cache. + _, err := c.secretInformer.Lister().Secrets(c.namespace).Get(certsSecretName) + notFound := k8serrors.IsNotFound(err) + if err != nil && !notFound { + return fmt.Errorf("failed to get %s/%s secret: %w", c.namespace, certsSecretName, err) + } + if !notFound { + // The secret already exists, so nothing to do. + return nil + } + + // Create a CA. + aggregatedAPIServerCA, err := certauthority.New(pkix.Name{CommonName: "Placeholder CA"}) + if err != nil { + return fmt.Errorf("could not initialize CA: %w", err) + } + + // This string must match the name of the Service declared in the deployment yaml. + const serviceName = "placeholder-name-api" + + // Using the CA from above, create a TLS server cert for the aggregated API server to use. + aggregatedAPIServerTLSCert, err := aggregatedAPIServerCA.Issue( + pkix.Name{CommonName: serviceName + "." + c.namespace + ".svc"}, + []string{}, + 24*365*time.Hour, + ) + if err != nil { + return fmt.Errorf("could not issue serving certificate: %w", err) + } + + // Write the CA's public key bundle and the serving certs to a secret. + tlsPrivateKeyPEM, tlsCertChainPEM, err := pemEncode(aggregatedAPIServerTLSCert) + if err != nil { + return fmt.Errorf("could not PEM encode serving certificate: %w", err) + } + secret := corev1.Secret{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{ + Name: certsSecretName, + Namespace: c.namespace, + }, + StringData: map[string]string{ + caCertificateSecretKey: string(aggregatedAPIServerCA.Bundle()), + tlsPrivateKeySecretKey: string(tlsPrivateKeyPEM), + tlsCertificateChainSecretKey: string(tlsCertChainPEM), + }, + } + _, err = c.k8sClient.CoreV1().Secrets(c.namespace).Create(ctx.Context, &secret, metav1.CreateOptions{}) + if err != nil { + return fmt.Errorf("could not create secret: %w", err) + } + + // Update the APIService to give it the new CA bundle. + if err := autoregistration.UpdateAPIService(ctx.Context, c.aggregatorClient, aggregatedAPIServerCA.Bundle()); err != nil { + return fmt.Errorf("could not update the API service: %w", err) + } + + klog.Info("certsManagerController Sync successfully created secret and updated API service") + return nil +} + +// Encode a tls.Certificate into a private key PEM and a cert chain PEM. +func pemEncode(cert *tls.Certificate) ([]byte, []byte, error) { + privateKeyDER, err := x509.MarshalPKCS8PrivateKey(cert.PrivateKey) + if err != nil { + return nil, nil, fmt.Errorf("error marshalling private key: %w", err) + } + privateKeyPEM := pem.EncodeToMemory(&pem.Block{ + Type: "PRIVATE KEY", + Headers: nil, + Bytes: privateKeyDER, + }) + + certChainPEM := make([]byte, 0) + for _, certFromChain := range cert.Certificate { + certPEMBytes := pem.EncodeToMemory(&pem.Block{ + Type: "CERTIFICATE", + Headers: nil, + Bytes: certFromChain, + }) + certChainPEM = append(certChainPEM, certPEMBytes...) + } + + return privateKeyPEM, certChainPEM, nil +} diff --git a/internal/controller/apicerts/certs_observer.go b/internal/controller/apicerts/certs_observer.go new file mode 100644 index 00000000..41595c91 --- /dev/null +++ b/internal/controller/apicerts/certs_observer.go @@ -0,0 +1,69 @@ +/* +Copyright 2020 VMware, Inc. +SPDX-License-Identifier: Apache-2.0 +*/ + +package apicerts + +import ( + "fmt" + + k8serrors "k8s.io/apimachinery/pkg/api/errors" + corev1informers "k8s.io/client-go/informers/core/v1" + "k8s.io/klog/v2" + + "github.com/suzerain-io/controller-go" + placeholdernamecontroller "github.com/suzerain-io/placeholder-name/internal/controller" + "github.com/suzerain-io/placeholder-name/internal/provider" +) + +type certsObserverController struct { + namespace string + dynamicCertProvider *provider.DynamicTLSServingCertProvider + secretInformer corev1informers.SecretInformer +} + +func NewCertsObserverController( + namespace string, + dynamicCertProvider *provider.DynamicTLSServingCertProvider, + secretInformer corev1informers.SecretInformer, + withInformer placeholdernamecontroller.WithInformerOptionFunc, +) controller.Controller { + return controller.New( + controller.Config{ + Name: "certs-observer-controller", + Syncer: &certsObserverController{ + namespace: namespace, + dynamicCertProvider: dynamicCertProvider, + secretInformer: secretInformer, + }, + }, + withInformer( + secretInformer, + placeholdernamecontroller.NameAndNamespaceExactMatchFilterFactory(certsSecretName, namespace), + controller.InformerOption{}, + ), + ) +} + +func (c *certsObserverController) Sync(_ controller.Context) error { + // Try to get the secret from the informer cache. + certSecret, err := c.secretInformer.Lister().Secrets(c.namespace).Get(certsSecretName) + notFound := k8serrors.IsNotFound(err) + if err != nil && !notFound { + return fmt.Errorf("failed to get %s/%s secret: %w", c.namespace, certsSecretName, err) + } + if notFound { + klog.Info("certsObserverController Sync() found that the secret does not exist yet or was deleted") + // The secret does not exist yet or was deleted. + c.dynamicCertProvider.CertPEM = nil + c.dynamicCertProvider.KeyPEM = nil + return nil + } + + // Mutate the in-memory cert provider to update with the latest cert values. + c.dynamicCertProvider.CertPEM = certSecret.Data[tlsCertificateChainSecretKey] + c.dynamicCertProvider.KeyPEM = certSecret.Data[tlsPrivateKeySecretKey] + klog.Info("certsObserverController Sync updated certs in the dynamic cert provider") + return nil +} diff --git a/internal/controller/logindiscovery/publisher.go b/internal/controller/logindiscovery/publisher.go index 4151a90c..54026c44 100644 --- a/internal/controller/logindiscovery/publisher.go +++ b/internal/controller/logindiscovery/publisher.go @@ -17,6 +17,7 @@ import ( "k8s.io/klog/v2" "github.com/suzerain-io/controller-go" + placeholdernamecontroller "github.com/suzerain-io/placeholder-name/internal/controller" crdsplaceholderv1alpha1 "github.com/suzerain-io/placeholder-name/kubernetes/1.19/api/apis/crdsplaceholder/v1alpha1" placeholderclientset "github.com/suzerain-io/placeholder-name/kubernetes/1.19/client-go/clientset/versioned" crdsplaceholderv1alpha1informers "github.com/suzerain-io/placeholder-name/kubernetes/1.19/client-go/informers/externalversions/crdsplaceholder/v1alpha1" @@ -31,25 +32,6 @@ const ( configName = "placeholder-name-config" ) -func nameAndNamespaceExactMatchFilterFactory(name, namespace string) controller.FilterFuncs { - objMatchesFunc := func(obj metav1.Object) bool { - return obj.GetName() == name && obj.GetNamespace() == namespace - } - return controller.FilterFuncs{ - AddFunc: objMatchesFunc, - UpdateFunc: func(oldObj, newObj metav1.Object) bool { - return objMatchesFunc(oldObj) || objMatchesFunc(newObj) - }, - DeleteFunc: objMatchesFunc, - } -} - -// Same signature as controller.WithInformer(). -type withInformerOptionFunc func( - getter controller.InformerGetter, - filter controller.Filter, - opt controller.InformerOption) controller.Option - type publisherController struct { namespace string serverOverride *string @@ -64,7 +46,7 @@ func NewPublisherController( placeholderClient placeholderclientset.Interface, configMapInformer corev1informers.ConfigMapInformer, loginDiscoveryConfigInformer crdsplaceholderv1alpha1informers.LoginDiscoveryConfigInformer, - withInformer withInformerOptionFunc, + withInformer placeholdernamecontroller.WithInformerOptionFunc, ) controller.Controller { return controller.New( controller.Config{ @@ -79,12 +61,12 @@ func NewPublisherController( }, withInformer( configMapInformer, - nameAndNamespaceExactMatchFilterFactory(clusterInfoName, ClusterInfoNamespace), + placeholdernamecontroller.NameAndNamespaceExactMatchFilterFactory(clusterInfoName, ClusterInfoNamespace), controller.InformerOption{}, ), withInformer( loginDiscoveryConfigInformer, - nameAndNamespaceExactMatchFilterFactory(configName, namespace), + placeholdernamecontroller.NameAndNamespaceExactMatchFilterFactory(configName, namespace), controller.InformerOption{}, ), ) diff --git a/internal/controller/utils.go b/internal/controller/utils.go new file mode 100644 index 00000000..1b47be1d --- /dev/null +++ b/internal/controller/utils.go @@ -0,0 +1,31 @@ +/* +Copyright 2020 VMware, Inc. +SPDX-License-Identifier: Apache-2.0 +*/ + +package controller + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/suzerain-io/controller-go" +) + +func NameAndNamespaceExactMatchFilterFactory(name, namespace string) controller.FilterFuncs { + objMatchesFunc := func(obj metav1.Object) bool { + return obj.GetName() == name && obj.GetNamespace() == namespace + } + return controller.FilterFuncs{ + AddFunc: objMatchesFunc, + UpdateFunc: func(oldObj, newObj metav1.Object) bool { + return objMatchesFunc(oldObj) || objMatchesFunc(newObj) + }, + DeleteFunc: objMatchesFunc, + } +} + +// Same signature as controller.WithInformer(). +type WithInformerOptionFunc func( + getter controller.InformerGetter, + filter controller.Filter, + opt controller.InformerOption) controller.Option diff --git a/internal/controller/prepare_controllers.go b/internal/controllermanager/prepare_controllers.go similarity index 56% rename from internal/controller/prepare_controllers.go rename to internal/controllermanager/prepare_controllers.go index 96d41ac5..8ab8eefa 100644 --- a/internal/controller/prepare_controllers.go +++ b/internal/controllermanager/prepare_controllers.go @@ -3,7 +3,7 @@ Copyright 2020 VMware, Inc. SPDX-License-Identifier: Apache-2.0 */ -package controller +package controllermanager import ( "context" @@ -14,11 +14,12 @@ import ( k8sinformers "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" restclient "k8s.io/client-go/rest" - aggregationv1client "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset" + aggregatorclient "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset" "github.com/suzerain-io/controller-go" - "github.com/suzerain-io/placeholder-name/internal/autoregistration" + "github.com/suzerain-io/placeholder-name/internal/controller/apicerts" "github.com/suzerain-io/placeholder-name/internal/controller/logindiscovery" + "github.com/suzerain-io/placeholder-name/internal/provider" placeholderclientset "github.com/suzerain-io/placeholder-name/kubernetes/1.19/client-go/clientset/versioned" placeholderinformers "github.com/suzerain-io/placeholder-name/kubernetes/1.19/client-go/informers/externalversions" ) @@ -30,25 +31,19 @@ const ( // Prepare the controllers and their informers and return a function that will start them when called. func PrepareControllers( - ctx context.Context, - caBundle []byte, serverInstallationNamespace string, discoveryURLOverride *string, + dynamicCertProvider *provider.DynamicTLSServingCertProvider, ) (func(ctx context.Context), error) { // Create k8s clients. - k8sClient, aggregationClient, placeholderClient, err := createClients() + k8sClient, aggregatorClient, placeholderClient, err := createClients() if err != nil { return nil, fmt.Errorf("could not create clients for the controllers: %w", err) } - // TODO Putting this here temporarily on the way toward moving it elsewhere. - // When it moves elsewhere then PrepareControllers() will not need to take ctx and caBundle parameters. - if err := autoregistration.UpdateAPIService(ctx, aggregationClient, caBundle); err != nil { - return nil, fmt.Errorf("could not update the API service: %w", err) - } - - // Create informers. - k8sInformers, placeholderInformers := createInformers(serverInstallationNamespace, k8sClient, placeholderClient) + // Create informers. Don't forget to make sure they get started in the function returned below. + kubePublicNamespaceK8sInformers, installationNamespaceK8sInformers, installationNamespacePlaceholderInformers := + createInformers(serverInstallationNamespace, k8sClient, placeholderClient) // Create controller manager. controllerManager := controller. @@ -58,8 +53,27 @@ func PrepareControllers( serverInstallationNamespace, discoveryURLOverride, placeholderClient, - k8sInformers.Core().V1().ConfigMaps(), - placeholderInformers.Crds().V1alpha1().LoginDiscoveryConfigs(), + kubePublicNamespaceK8sInformers.Core().V1().ConfigMaps(), + installationNamespacePlaceholderInformers.Crds().V1alpha1().LoginDiscoveryConfigs(), + controller.WithInformer, + ), + singletonWorker, + ). + WithController( + apicerts.NewCertsManagerController( + serverInstallationNamespace, + k8sClient, + aggregatorClient, + installationNamespaceK8sInformers.Core().V1().Secrets(), + controller.WithInformer, + ), + singletonWorker, + ). + WithController( + apicerts.NewCertsObserverController( + serverInstallationNamespace, + dynamicCertProvider, + installationNamespaceK8sInformers.Core().V1().Secrets(), controller.WithInformer, ), singletonWorker, @@ -67,14 +81,21 @@ func PrepareControllers( // Return a function which starts the informers and controllers. return func(ctx context.Context) { - k8sInformers.Start(ctx.Done()) - placeholderInformers.Start(ctx.Done()) + kubePublicNamespaceK8sInformers.Start(ctx.Done()) + installationNamespaceK8sInformers.Start(ctx.Done()) + installationNamespacePlaceholderInformers.Start(ctx.Done()) + go controllerManager.Start(ctx) }, nil } // Create the k8s clients that will be used by the controllers. -func createClients() (*kubernetes.Clientset, *aggregationv1client.Clientset, *placeholderclientset.Clientset, error) { +func createClients() ( + k8sClient *kubernetes.Clientset, + aggregatorClient *aggregatorclient.Clientset, + placeholderClient *placeholderclientset.Clientset, + err error, +) { // Load the Kubernetes client configuration (kubeconfig), kubeConfig, err := restclient.InClusterConfig() if err != nil { @@ -85,13 +106,13 @@ func createClients() (*kubernetes.Clientset, *aggregationv1client.Clientset, *pl protoKubeConfig := createProtoKubeConfig(kubeConfig) // Connect to the core Kubernetes API. - k8sClient, err := kubernetes.NewForConfig(protoKubeConfig) + k8sClient, err = kubernetes.NewForConfig(protoKubeConfig) if err != nil { return nil, nil, nil, fmt.Errorf("could not initialize Kubernetes client: %w", err) } // Connect to the Kubernetes aggregation API. - aggregationClient, err := aggregationv1client.NewForConfig(protoKubeConfig) + aggregatorClient, err = aggregatorclient.NewForConfig(protoKubeConfig) if err != nil { return nil, nil, nil, fmt.Errorf("could not initialize Kubernetes client: %w", err) } @@ -99,12 +120,13 @@ func createClients() (*kubernetes.Clientset, *aggregationv1client.Clientset, *pl // Connect to the placeholder API. // I think we can't use protobuf encoding here because we are using CRDs // (for which protobuf encoding is not supported). - placeholderClient, err := placeholderclientset.NewForConfig(kubeConfig) + placeholderClient, err = placeholderclientset.NewForConfig(kubeConfig) if err != nil { return nil, nil, nil, fmt.Errorf("could not initialize placeholder client: %w", err) } - return k8sClient, aggregationClient, placeholderClient, nil + //nolint: nakedret + return } // Create the informers that will be used by the controllers. @@ -112,20 +134,27 @@ func createInformers( serverInstallationNamespace string, k8sClient *kubernetes.Clientset, placeholderClient *placeholderclientset.Clientset, -) (k8sinformers.SharedInformerFactory, placeholderinformers.SharedInformerFactory) { - k8sInformers := k8sinformers.NewSharedInformerFactoryWithOptions( +) ( + kubePublicNamespaceK8sInformers k8sinformers.SharedInformerFactory, + installationNamespaceK8sInformers k8sinformers.SharedInformerFactory, + installationNamespacePlaceholderInformers placeholderinformers.SharedInformerFactory, +) { + kubePublicNamespaceK8sInformers = k8sinformers.NewSharedInformerFactoryWithOptions( k8sClient, defaultResyncInterval, - k8sinformers.WithNamespace( - logindiscovery.ClusterInfoNamespace, - ), + k8sinformers.WithNamespace(logindiscovery.ClusterInfoNamespace), ) - placeholderInformers := placeholderinformers.NewSharedInformerFactoryWithOptions( + installationNamespaceK8sInformers = k8sinformers.NewSharedInformerFactoryWithOptions( + k8sClient, + defaultResyncInterval, + k8sinformers.WithNamespace(serverInstallationNamespace), + ) + installationNamespacePlaceholderInformers = placeholderinformers.NewSharedInformerFactoryWithOptions( placeholderClient, defaultResyncInterval, placeholderinformers.WithNamespace(serverInstallationNamespace), ) - return k8sInformers, placeholderInformers + return } // Returns a copy of the input config with the ContentConfig set to use protobuf. diff --git a/internal/provider/dynamic_tls_serving_cert_provider.go b/internal/provider/dynamic_tls_serving_cert_provider.go new file mode 100644 index 00000000..7807802e --- /dev/null +++ b/internal/provider/dynamic_tls_serving_cert_provider.go @@ -0,0 +1,19 @@ +/* +Copyright 2020 VMware, Inc. +SPDX-License-Identifier: Apache-2.0 +*/ + +package provider + +type DynamicTLSServingCertProvider struct { + CertPEM []byte + KeyPEM []byte +} + +func (*DynamicTLSServingCertProvider) Name() string { + return "DynamicTLSServingCertProvider" +} + +func (p *DynamicTLSServingCertProvider) CurrentCertKeyContent() (cert []byte, key []byte) { + return p.CertPEM, p.KeyPEM +} diff --git a/internal/server/server.go b/internal/server/server.go index 6ec828ce..68cf2a1e 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -8,24 +8,19 @@ package server import ( "context" - "crypto/tls" - "crypto/x509" - "crypto/x509/pkix" - "encoding/pem" "fmt" "io" - "time" "github.com/spf13/cobra" genericapiserver "k8s.io/apiserver/pkg/server" - "k8s.io/apiserver/pkg/server/dynamiccertificates" genericoptions "k8s.io/apiserver/pkg/server/options" "k8s.io/apiserver/plugin/pkg/authenticator/token/webhook" "github.com/suzerain-io/placeholder-name/internal/apiserver" "github.com/suzerain-io/placeholder-name/internal/certauthority" - "github.com/suzerain-io/placeholder-name/internal/controller" + "github.com/suzerain-io/placeholder-name/internal/controllermanager" "github.com/suzerain-io/placeholder-name/internal/downward" + "github.com/suzerain-io/placeholder-name/internal/provider" placeholderv1alpha1 "github.com/suzerain-io/placeholder-name/kubernetes/1.19/api/apis/placeholder/v1alpha1" "github.com/suzerain-io/placeholder-name/pkg/config" ) @@ -135,31 +130,19 @@ func (app *App) runServer(ctx context.Context) error { } serverInstallationNamespace := podInfo.Namespace - // Create a CA. - aggregatedAPIServerCA, err := certauthority.New(pkix.Name{CommonName: "Placeholder CA"}) - if err != nil { - return fmt.Errorf("could not initialize CA: %w", err) - } - - // This string must match the name of the Service declared in the deployment yaml. - const serviceName = "placeholder-name-api" - // Using the CA from above, create a TLS server cert for the aggregated API server to use. - aggregatedAPIServerTLSCert, err := aggregatedAPIServerCA.Issue( - pkix.Name{CommonName: serviceName + "." + serverInstallationNamespace + ".svc"}, - []string{}, - 24*365*time.Hour, - ) - if err != nil { - return fmt.Errorf("could not issue serving certificate: %w", err) - } + // This cert provider will provide certs to the API server and will + // be mutated by a controller to keep the certs up to date with what + // is stored in a k8s Secret. Therefore it also effectively acting as + // an in-memory cache of what is stored in the k8s Secret, helping to + // keep incoming requests fast. + dynamicCertProvider := &provider.DynamicTLSServingCertProvider{} // Prepare to start the controllers, but defer actually starting them until the // post start hook of the aggregated API server. - startControllersFunc, err := controller.PrepareControllers( - ctx, - aggregatedAPIServerCA.Bundle(), + startControllersFunc, err := controllermanager.PrepareControllers( serverInstallationNamespace, cfg.DiscoveryConfig.URL, + dynamicCertProvider, ) if err != nil { return fmt.Errorf("could not prepare controllers: %w", err) @@ -167,7 +150,7 @@ func (app *App) runServer(ctx context.Context) error { // Get the aggregated API server config. aggregatedAPIServerConfig, err := getAggregatedAPIServerConfig( - aggregatedAPIServerTLSCert, + dynamicCertProvider, webhookTokenAuthenticator, k8sClusterCA, startControllersFunc, @@ -188,25 +171,29 @@ func (app *App) runServer(ctx context.Context) error { // Create a configuration for the aggregated API server. func getAggregatedAPIServerConfig( - cert *tls.Certificate, + dynamicCertProvider *provider.DynamicTLSServingCertProvider, webhookTokenAuthenticator *webhook.WebhookTokenAuthenticator, ca *certauthority.CA, startControllersPostStartHook func(context.Context), ) (*apiserver.Config, error) { - provider, err := createStaticCertKeyProvider(cert) - if err != nil { - return nil, fmt.Errorf("could not create static cert key provider: %w", err) - } - recommendedOptions := genericoptions.NewRecommendedOptions( defaultEtcdPathPrefix, apiserver.Codecs.LegacyCodec(placeholderv1alpha1.SchemeGroupVersion), // TODO we should check to see if all the other default settings are acceptable for us ) recommendedOptions.Etcd = nil // turn off etcd storage because we don't need it yet - recommendedOptions.SecureServing.ServerCert.GeneratedCert = provider + recommendedOptions.SecureServing.ServerCert.GeneratedCert = dynamicCertProvider serverConfig := genericapiserver.NewRecommendedConfig(apiserver.Codecs) + // Note that among other things, this ApplyTo() function copies + // `recommendedOptions.SecureServing.ServerCert.GeneratedCert` into + // `serverConfig.SecureServing.Cert` thus making `dynamicCertProvider` + // the cert provider for the running server. The provider will be called + // by the API machinery periodically. When the provider returns nil certs, + // the API server will return "the server is currently unable to + // handle the request" error responses for all incoming requests. + // If the provider later starts returning certs, then the API server + // will use them to handle the incoming requests successfully. if err := recommendedOptions.ApplyTo(serverConfig); err != nil { return nil, err } @@ -221,27 +208,3 @@ func getAggregatedAPIServerConfig( } return apiServerConfig, nil } - -func createStaticCertKeyProvider(cert *tls.Certificate) (dynamiccertificates.CertKeyContentProvider, error) { - privateKeyDER, err := x509.MarshalPKCS8PrivateKey(cert.PrivateKey) - if err != nil { - return nil, fmt.Errorf("error marshalling private key: %w", err) - } - privateKeyPEM := pem.EncodeToMemory(&pem.Block{ - Type: "PRIVATE KEY", - Headers: nil, - Bytes: privateKeyDER, - }) - - certChainPEM := make([]byte, 0) - for _, certFromChain := range cert.Certificate { - certPEMBytes := pem.EncodeToMemory(&pem.Block{ - Type: "CERTIFICATE", - Headers: nil, - Bytes: certFromChain, - }) - certChainPEM = append(certChainPEM, certPEMBytes...) - } - - return dynamiccertificates.NewStaticCertKeyContent("some-name???", certChainPEM, privateKeyPEM) -}