First draft of moving API server TLS cert generation to controllers

- Refactors the existing cert generation code into controllers
  which read and write a Secret containing the certs
- Does not add any new functionality yet, e.g. no new handling
  for cert expiration, and no leader election to allow for
  multiple servers running simultaneously
- This commit also doesn't add new tests for the cert generation
  code, but it should be more unit testable now as controllers

Dockerfile

@@ -1,3 +1,6 @@
+# Copyright 2020 VMware, Inc.
+# SPDX-License-Identifier: Apache-2.0
+
 FROM golang:1.14-alpine as build-env
 
 # It is important that these ARG's are defined after the FROM statement

README.md

@@ -1 +1,3 @@
 # placeholder-name
+
+Copyright 2020 VMware, Inc.

deploy/rbac.yaml

@@ -1,5 +1,6 @@
 #@ load("@ytt:data", "data")
 
+#! Give permission to various cluster-scoped objects
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
@@ -28,6 +29,8 @@ roleRef:
   kind: ClusterRole
   name: #@ data.values.app_name + "-aggregated-api-server-cluster-role"
   apiGroup: rbac.authorization.k8s.io
+
+#! Give permission to various objects within the app's own namespace
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
@@ -38,6 +41,9 @@ rules:
   - apiGroups: [""]
     resources: [services]
     verbs: [create, get, list, patch, update, watch]
+  - apiGroups: [""]
+    resources: [secrets]
+    verbs: [create, get, list, patch, update, watch]
   - apiGroups: [crds.placeholder.suzerain-io.github.io]
     resources: [logindiscoveryconfigs]
     verbs: [create, get, list, update, watch]
@@ -55,6 +61,8 @@ roleRef:
   kind: Role
   name: #@ data.values.app_name + "-aggregated-api-server-role"
   apiGroup: rbac.authorization.k8s.io
+
+#! Allow both authenticated and unauthenticated LoginRequests (i.e. allow all requests)
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
@@ -70,7 +78,6 @@ apiVersion: rbac.authorization.k8s.io/v1
 metadata:
   name: #@ data.values.app_name + "-loginrequests-cluster-role-binding"
 subjects:
-  #! both authenticated and unauthenticated requests (i.e. all requests) should be allowed
   - kind: Group
     name: system:authenticated
     apiGroup: rbac.authorization.k8s.io
@@ -81,6 +88,8 @@ roleRef:
   kind: ClusterRole
   name: #@ data.values.app_name + "-loginrequests-cluster-role"
   apiGroup: rbac.authorization.k8s.io
+
+#! Give permissions for subjectaccessreviews, tokenreview that is needed by aggregated api servers
 ---
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
@@ -93,9 +102,10 @@ subjects:
     namespace: #@ data.values.namespace
 roleRef:
   kind: ClusterRole
-  #! give permissions for subjectaccessreviews, tokenreview that is needed by aggregated api servers
   name: system:auth-delegator
   apiGroup: rbac.authorization.k8s.io
+
+#! Give permissions for a special configmap of CA bundles that is needed by aggregated api servers
 ---
 kind: RoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
@@ -108,9 +118,10 @@ subjects:
     namespace: #@ data.values.namespace
 roleRef:
   kind: Role
-  #! give permissions for a special configmap of CA bundles that is needed by aggregated api servers
   name: extension-apiserver-authentication-reader
   apiGroup: rbac.authorization.k8s.io
+
+#! Give permission to list and watch ConfigMaps in kube-public
 ---
 kind: Role
 apiVersion: rbac.authorization.k8s.io/v1

internal/autoregistration/autoregistration.go

@@ -12,14 +12,14 @@ import (
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/util/retry"
-	aggregatationv1client "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset"
+	aggregatorclient "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset"
 
 	placeholderv1alpha1 "github.com/suzerain-io/placeholder-name/kubernetes/1.19/api/apis/placeholder/v1alpha1"
 )
 
 // UpdateAPIService updates the APIService's CA bundle.
-func UpdateAPIService(ctx context.Context, aggregationV1 aggregatationv1client.Interface, aggregatedAPIServerCA []byte) error {
+func UpdateAPIService(ctx context.Context, aggregatorClient aggregatorclient.Interface, aggregatedAPIServerCA []byte) error {
-	apiServices := aggregationV1.ApiregistrationV1().APIServices()
+	apiServices := aggregatorClient.ApiregistrationV1().APIServices()
 	apiServiceName := placeholderv1alpha1.SchemeGroupVersion.Version + "." + placeholderv1alpha1.GroupName
 
 	if err := retry.RetryOnConflict(retry.DefaultRetry, func() error {

internal/controller/apicerts/certs_manager.go

@@ -0,0 +1,164 @@
+/*
+Copyright 2020 VMware, Inc.
+SPDX-License-Identifier: Apache-2.0
+*/
+
+package apicerts
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"fmt"
+	"time"
+
+	corev1 "k8s.io/api/core/v1"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	corev1informers "k8s.io/client-go/informers/core/v1"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/klog/v2"
+	aggregatorclient "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset"
+
+	"github.com/suzerain-io/controller-go"
+	"github.com/suzerain-io/placeholder-name/internal/autoregistration"
+	"github.com/suzerain-io/placeholder-name/internal/certauthority"
+	placeholdernamecontroller "github.com/suzerain-io/placeholder-name/internal/controller"
+	placeholderv1alpha1 "github.com/suzerain-io/placeholder-name/kubernetes/1.19/api/apis/placeholder/v1alpha1"
+)
+
+const (
+	//nolint: gosec
+	certsSecretName              = "api-serving-cert"
+	caCertificateSecretKey       = "caCertificate"
+	tlsPrivateKeySecretKey       = "tlsPrivateKey"
+	tlsCertificateChainSecretKey = "tlsCertificateChain"
+)
+
+type certsManagerController struct {
+	namespace        string
+	apiServiceName   string
+	k8sClient        kubernetes.Interface
+	aggregatorClient *aggregatorclient.Clientset
+	secretInformer   corev1informers.SecretInformer
+}
+
+func NewCertsManagerController(
+	namespace string,
+	k8sClient kubernetes.Interface,
+	aggregationClient *aggregatorclient.Clientset,
+	secretInformer corev1informers.SecretInformer,
+	withInformer placeholdernamecontroller.WithInformerOptionFunc,
+) controller.Controller {
+	apiServiceName := placeholderv1alpha1.SchemeGroupVersion.Version + "." + placeholderv1alpha1.GroupName
+	return controller.New(
+		controller.Config{
+			Name: "certs-manager-controller",
+			Syncer: &certsManagerController{
+				apiServiceName:   apiServiceName,
+				namespace:        namespace,
+				k8sClient:        k8sClient,
+				aggregatorClient: aggregationClient,
+				secretInformer:   secretInformer,
+			},
+		},
+		withInformer(
+			secretInformer,
+			placeholdernamecontroller.NameAndNamespaceExactMatchFilterFactory(certsSecretName, namespace),
+			controller.InformerOption{},
+		),
+		// Be sure to run once even if the Secret that the informer is watching doesn't exist.
+		controller.WithInitialEvent(controller.Key{
+			Namespace: namespace,
+			Name:      certsSecretName,
+		}),
+	)
+}
+
+func (c *certsManagerController) Sync(ctx controller.Context) error {
+	// Try to get the secret from the informer cache.
+	_, err := c.secretInformer.Lister().Secrets(c.namespace).Get(certsSecretName)
+	notFound := k8serrors.IsNotFound(err)
+	if err != nil && !notFound {
+		return fmt.Errorf("failed to get %s/%s secret: %w", c.namespace, certsSecretName, err)
+	}
+	if !notFound {
+		// The secret already exists, so nothing to do.
+		return nil
+	}
+
+	// Create a CA.
+	aggregatedAPIServerCA, err := certauthority.New(pkix.Name{CommonName: "Placeholder CA"})
+	if err != nil {
+		return fmt.Errorf("could not initialize CA: %w", err)
+	}
+
+	// This string must match the name of the Service declared in the deployment yaml.
+	const serviceName = "placeholder-name-api"
+
+	// Using the CA from above, create a TLS server cert for the aggregated API server to use.
+	aggregatedAPIServerTLSCert, err := aggregatedAPIServerCA.Issue(
+		pkix.Name{CommonName: serviceName + "." + c.namespace + ".svc"},
+		[]string{},
+		24*365*time.Hour,
+	)
+	if err != nil {
+		return fmt.Errorf("could not issue serving certificate: %w", err)
+	}
+
+	// Write the CA's public key bundle and the serving certs to a secret.
+	tlsPrivateKeyPEM, tlsCertChainPEM, err := pemEncode(aggregatedAPIServerTLSCert)
+	if err != nil {
+		return fmt.Errorf("could not PEM encode serving certificate: %w", err)
+	}
+	secret := corev1.Secret{
+		TypeMeta: metav1.TypeMeta{},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      certsSecretName,
+			Namespace: c.namespace,
+		},
+		StringData: map[string]string{
+			caCertificateSecretKey:       string(aggregatedAPIServerCA.Bundle()),
+			tlsPrivateKeySecretKey:       string(tlsPrivateKeyPEM),
+			tlsCertificateChainSecretKey: string(tlsCertChainPEM),
+		},
+	}
+	_, err = c.k8sClient.CoreV1().Secrets(c.namespace).Create(ctx.Context, &secret, metav1.CreateOptions{})
+	if err != nil {
+		return fmt.Errorf("could not create secret: %w", err)
+	}
+
+	// Update the APIService to give it the new CA bundle.
+	if err := autoregistration.UpdateAPIService(ctx.Context, c.aggregatorClient, aggregatedAPIServerCA.Bundle()); err != nil {
+		return fmt.Errorf("could not update the API service: %w", err)
+	}
+
+	klog.Info("certsManagerController Sync successfully created secret and updated API service")
+	return nil
+}
+
+// Encode a tls.Certificate into a private key PEM and a cert chain PEM.
+func pemEncode(cert *tls.Certificate) ([]byte, []byte, error) {
+	privateKeyDER, err := x509.MarshalPKCS8PrivateKey(cert.PrivateKey)
+	if err != nil {
+		return nil, nil, fmt.Errorf("error marshalling private key: %w", err)
+	}
+	privateKeyPEM := pem.EncodeToMemory(&pem.Block{
+		Type:    "PRIVATE KEY",
+		Headers: nil,
+		Bytes:   privateKeyDER,
+	})
+
+	certChainPEM := make([]byte, 0)
+	for _, certFromChain := range cert.Certificate {
+		certPEMBytes := pem.EncodeToMemory(&pem.Block{
+			Type:    "CERTIFICATE",
+			Headers: nil,
+			Bytes:   certFromChain,
+		})
+		certChainPEM = append(certChainPEM, certPEMBytes...)
+	}
+
+	return privateKeyPEM, certChainPEM, nil
+}

internal/controller/apicerts/certs_observer.go

@@ -0,0 +1,69 @@
+/*
+Copyright 2020 VMware, Inc.
+SPDX-License-Identifier: Apache-2.0
+*/
+
+package apicerts
+
+import (
+	"fmt"
+
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	corev1informers "k8s.io/client-go/informers/core/v1"
+	"k8s.io/klog/v2"
+
+	"github.com/suzerain-io/controller-go"
+	placeholdernamecontroller "github.com/suzerain-io/placeholder-name/internal/controller"
+	"github.com/suzerain-io/placeholder-name/internal/provider"
+)
+
+type certsObserverController struct {
+	namespace           string
+	dynamicCertProvider *provider.DynamicTLSServingCertProvider
+	secretInformer      corev1informers.SecretInformer
+}
+
+func NewCertsObserverController(
+	namespace string,
+	dynamicCertProvider *provider.DynamicTLSServingCertProvider,
+	secretInformer corev1informers.SecretInformer,
+	withInformer placeholdernamecontroller.WithInformerOptionFunc,
+) controller.Controller {
+	return controller.New(
+		controller.Config{
+			Name: "certs-observer-controller",
+			Syncer: &certsObserverController{
+				namespace:           namespace,
+				dynamicCertProvider: dynamicCertProvider,
+				secretInformer:      secretInformer,
+			},
+		},
+		withInformer(
+			secretInformer,
+			placeholdernamecontroller.NameAndNamespaceExactMatchFilterFactory(certsSecretName, namespace),
+			controller.InformerOption{},
+		),
+	)
+}
+
+func (c *certsObserverController) Sync(_ controller.Context) error {
+	// Try to get the secret from the informer cache.
+	certSecret, err := c.secretInformer.Lister().Secrets(c.namespace).Get(certsSecretName)
+	notFound := k8serrors.IsNotFound(err)
+	if err != nil && !notFound {
+		return fmt.Errorf("failed to get %s/%s secret: %w", c.namespace, certsSecretName, err)
+	}
+	if notFound {
+		klog.Info("certsObserverController Sync() found that the secret does not exist yet or was deleted")
+		// The secret does not exist yet or was deleted.
+		c.dynamicCertProvider.CertPEM = nil
+		c.dynamicCertProvider.KeyPEM = nil
+		return nil
+	}
+
+	// Mutate the in-memory cert provider to update with the latest cert values.
+	c.dynamicCertProvider.CertPEM = certSecret.Data[tlsCertificateChainSecretKey]
+	c.dynamicCertProvider.KeyPEM = certSecret.Data[tlsPrivateKeySecretKey]
+	klog.Info("certsObserverController Sync updated certs in the dynamic cert provider")
+	return nil
+}

internal/controller/logindiscovery/publisher.go

@@ -17,6 +17,7 @@ import (
 	"k8s.io/klog/v2"
 
 	"github.com/suzerain-io/controller-go"
+	placeholdernamecontroller "github.com/suzerain-io/placeholder-name/internal/controller"
 	crdsplaceholderv1alpha1 "github.com/suzerain-io/placeholder-name/kubernetes/1.19/api/apis/crdsplaceholder/v1alpha1"
 	placeholderclientset "github.com/suzerain-io/placeholder-name/kubernetes/1.19/client-go/clientset/versioned"
 	crdsplaceholderv1alpha1informers "github.com/suzerain-io/placeholder-name/kubernetes/1.19/client-go/informers/externalversions/crdsplaceholder/v1alpha1"
@@ -31,25 +32,6 @@ const (
 	configName = "placeholder-name-config"
 )
 
-func nameAndNamespaceExactMatchFilterFactory(name, namespace string) controller.FilterFuncs {
-	objMatchesFunc := func(obj metav1.Object) bool {
-		return obj.GetName() == name && obj.GetNamespace() == namespace
-	}
-	return controller.FilterFuncs{
-		AddFunc: objMatchesFunc,
-		UpdateFunc: func(oldObj, newObj metav1.Object) bool {
-			return objMatchesFunc(oldObj) || objMatchesFunc(newObj)
-		},
-		DeleteFunc: objMatchesFunc,
-	}
-}
-
-// Same signature as controller.WithInformer().
-type withInformerOptionFunc func(
-	getter controller.InformerGetter,
-	filter controller.Filter,
-	opt controller.InformerOption) controller.Option
-
 type publisherController struct {
 	namespace                    string
 	serverOverride               *string
@@ -64,7 +46,7 @@ func NewPublisherController(
 	placeholderClient placeholderclientset.Interface,
 	configMapInformer corev1informers.ConfigMapInformer,
 	loginDiscoveryConfigInformer crdsplaceholderv1alpha1informers.LoginDiscoveryConfigInformer,
-	withInformer withInformerOptionFunc,
+	withInformer placeholdernamecontroller.WithInformerOptionFunc,
 ) controller.Controller {
 	return controller.New(
 		controller.Config{
@@ -79,12 +61,12 @@ func NewPublisherController(
 		},
 		withInformer(
 			configMapInformer,
-			nameAndNamespaceExactMatchFilterFactory(clusterInfoName, ClusterInfoNamespace),
+			placeholdernamecontroller.NameAndNamespaceExactMatchFilterFactory(clusterInfoName, ClusterInfoNamespace),
 			controller.InformerOption{},
 		),
 		withInformer(
 			loginDiscoveryConfigInformer,
-			nameAndNamespaceExactMatchFilterFactory(configName, namespace),
+			placeholdernamecontroller.NameAndNamespaceExactMatchFilterFactory(configName, namespace),
 			controller.InformerOption{},
 		),
 	)

internal/controller/utils.go

@@ -0,0 +1,31 @@
+/*
+Copyright 2020 VMware, Inc.
+SPDX-License-Identifier: Apache-2.0
+*/
+
+package controller
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/suzerain-io/controller-go"
+)
+
+func NameAndNamespaceExactMatchFilterFactory(name, namespace string) controller.FilterFuncs {
+	objMatchesFunc := func(obj metav1.Object) bool {
+		return obj.GetName() == name && obj.GetNamespace() == namespace
+	}
+	return controller.FilterFuncs{
+		AddFunc: objMatchesFunc,
+		UpdateFunc: func(oldObj, newObj metav1.Object) bool {
+			return objMatchesFunc(oldObj) || objMatchesFunc(newObj)
+		},
+		DeleteFunc: objMatchesFunc,
+	}
+}
+
+// Same signature as controller.WithInformer().
+type WithInformerOptionFunc func(
+	getter controller.InformerGetter,
+	filter controller.Filter,
+	opt controller.InformerOption) controller.Option

internal/controllermanager/prepare_controllers.go

@@ -3,7 +3,7 @@ Copyright 2020 VMware, Inc.
 SPDX-License-Identifier: Apache-2.0
 */
 
-package controller
+package controllermanager
 
 import (
 	"context"
@@ -14,11 +14,12 @@ import (
 	k8sinformers "k8s.io/client-go/informers"
 	"k8s.io/client-go/kubernetes"
 	restclient "k8s.io/client-go/rest"
-	aggregationv1client "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset"
+	aggregatorclient "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset"
 
 	"github.com/suzerain-io/controller-go"
-	"github.com/suzerain-io/placeholder-name/internal/autoregistration"
+	"github.com/suzerain-io/placeholder-name/internal/controller/apicerts"
 	"github.com/suzerain-io/placeholder-name/internal/controller/logindiscovery"
+	"github.com/suzerain-io/placeholder-name/internal/provider"
 	placeholderclientset "github.com/suzerain-io/placeholder-name/kubernetes/1.19/client-go/clientset/versioned"
 	placeholderinformers "github.com/suzerain-io/placeholder-name/kubernetes/1.19/client-go/informers/externalversions"
 )
@@ -30,25 +31,19 @@ const (
 
 // Prepare the controllers and their informers and return a function that will start them when called.
 func PrepareControllers(
-	ctx context.Context,
-	caBundle []byte,
 	serverInstallationNamespace string,
 	discoveryURLOverride *string,
+	dynamicCertProvider *provider.DynamicTLSServingCertProvider,
 ) (func(ctx context.Context), error) {
 	// Create k8s clients.
-	k8sClient, aggregationClient, placeholderClient, err := createClients()
+	k8sClient, aggregatorClient, placeholderClient, err := createClients()
 	if err != nil {
 		return nil, fmt.Errorf("could not create clients for the controllers: %w", err)
 	}
 
-	// TODO Putting this here temporarily on the way toward moving it elsewhere.
+	// Create informers. Don't forget to make sure they get started in the function returned below.
-	//  When it moves elsewhere then PrepareControllers() will not need to take ctx and caBundle parameters.
+	kubePublicNamespaceK8sInformers, installationNamespaceK8sInformers, installationNamespacePlaceholderInformers :=
-	if err := autoregistration.UpdateAPIService(ctx, aggregationClient, caBundle); err != nil {
+		createInformers(serverInstallationNamespace, k8sClient, placeholderClient)
-		return nil, fmt.Errorf("could not update the API service: %w", err)
-	}
-
-	// Create informers.
-	k8sInformers, placeholderInformers := createInformers(serverInstallationNamespace, k8sClient, placeholderClient)
 
 	// Create controller manager.
 	controllerManager := controller.
@@ -58,8 +53,27 @@ func PrepareControllers(
 				serverInstallationNamespace,
 				discoveryURLOverride,
 				placeholderClient,
-				k8sInformers.Core().V1().ConfigMaps(),
+				kubePublicNamespaceK8sInformers.Core().V1().ConfigMaps(),
-				placeholderInformers.Crds().V1alpha1().LoginDiscoveryConfigs(),
+				installationNamespacePlaceholderInformers.Crds().V1alpha1().LoginDiscoveryConfigs(),
+				controller.WithInformer,
+			),
+			singletonWorker,
+		).
+		WithController(
+			apicerts.NewCertsManagerController(
+				serverInstallationNamespace,
+				k8sClient,
+				aggregatorClient,
+				installationNamespaceK8sInformers.Core().V1().Secrets(),
+				controller.WithInformer,
+			),
+			singletonWorker,
+		).
+		WithController(
+			apicerts.NewCertsObserverController(
+				serverInstallationNamespace,
+				dynamicCertProvider,
+				installationNamespaceK8sInformers.Core().V1().Secrets(),
 				controller.WithInformer,
 			),
 			singletonWorker,
@@ -67,14 +81,21 @@ func PrepareControllers(
 
 	// Return a function which starts the informers and controllers.
 	return func(ctx context.Context) {
-		k8sInformers.Start(ctx.Done())
+		kubePublicNamespaceK8sInformers.Start(ctx.Done())
-		placeholderInformers.Start(ctx.Done())
+		installationNamespaceK8sInformers.Start(ctx.Done())
+		installationNamespacePlaceholderInformers.Start(ctx.Done())
+
 		go controllerManager.Start(ctx)
 	}, nil
 }
 
 // Create the k8s clients that will be used by the controllers.
-func createClients() (*kubernetes.Clientset, *aggregationv1client.Clientset, *placeholderclientset.Clientset, error) {
+func createClients() (
+	k8sClient *kubernetes.Clientset,
+	aggregatorClient *aggregatorclient.Clientset,
+	placeholderClient *placeholderclientset.Clientset,
+	err error,
+) {
 	// Load the Kubernetes client configuration (kubeconfig),
 	kubeConfig, err := restclient.InClusterConfig()
 	if err != nil {
@@ -85,13 +106,13 @@ func createClients() (*kubernetes.Clientset, *aggregationv1client.Clientset, *pl
 	protoKubeConfig := createProtoKubeConfig(kubeConfig)
 
 	// Connect to the core Kubernetes API.
-	k8sClient, err := kubernetes.NewForConfig(protoKubeConfig)
+	k8sClient, err = kubernetes.NewForConfig(protoKubeConfig)
 	if err != nil {
 		return nil, nil, nil, fmt.Errorf("could not initialize Kubernetes client: %w", err)
 	}
 
 	// Connect to the Kubernetes aggregation API.
-	aggregationClient, err := aggregationv1client.NewForConfig(protoKubeConfig)
+	aggregatorClient, err = aggregatorclient.NewForConfig(protoKubeConfig)
 	if err != nil {
 		return nil, nil, nil, fmt.Errorf("could not initialize Kubernetes client: %w", err)
 	}
@@ -99,12 +120,13 @@ func createClients() (*kubernetes.Clientset, *aggregationv1client.Clientset, *pl
 	// Connect to the placeholder API.
 	// I think we can't use protobuf encoding here because we are using CRDs
 	// (for which protobuf encoding is not supported).
-	placeholderClient, err := placeholderclientset.NewForConfig(kubeConfig)
+	placeholderClient, err = placeholderclientset.NewForConfig(kubeConfig)
 	if err != nil {
 		return nil, nil, nil, fmt.Errorf("could not initialize placeholder client: %w", err)
 	}
 
-	return k8sClient, aggregationClient, placeholderClient, nil
+	//nolint: nakedret
+	return
 }
 
 // Create the informers that will be used by the controllers.
@@ -112,20 +134,27 @@ func createInformers(
 	serverInstallationNamespace string,
 	k8sClient *kubernetes.Clientset,
 	placeholderClient *placeholderclientset.Clientset,
-) (k8sinformers.SharedInformerFactory, placeholderinformers.SharedInformerFactory) {
+) (
-	k8sInformers := k8sinformers.NewSharedInformerFactoryWithOptions(
+	kubePublicNamespaceK8sInformers k8sinformers.SharedInformerFactory,
+	installationNamespaceK8sInformers k8sinformers.SharedInformerFactory,
+	installationNamespacePlaceholderInformers placeholderinformers.SharedInformerFactory,
+) {
+	kubePublicNamespaceK8sInformers = k8sinformers.NewSharedInformerFactoryWithOptions(
 		k8sClient,
 		defaultResyncInterval,
-		k8sinformers.WithNamespace(
+		k8sinformers.WithNamespace(logindiscovery.ClusterInfoNamespace),
-			logindiscovery.ClusterInfoNamespace,
-		),
 	)
-	placeholderInformers := placeholderinformers.NewSharedInformerFactoryWithOptions(
+	installationNamespaceK8sInformers = k8sinformers.NewSharedInformerFactoryWithOptions(
+		k8sClient,
+		defaultResyncInterval,
+		k8sinformers.WithNamespace(serverInstallationNamespace),
+	)
+	installationNamespacePlaceholderInformers = placeholderinformers.NewSharedInformerFactoryWithOptions(
 		placeholderClient,
 		defaultResyncInterval,
 		placeholderinformers.WithNamespace(serverInstallationNamespace),
 	)
-	return k8sInformers, placeholderInformers
+	return
 }
 
 // Returns a copy of the input config with the ContentConfig set to use protobuf.

internal/provider/dynamic_tls_serving_cert_provider.go

@@ -0,0 +1,19 @@
+/*
+Copyright 2020 VMware, Inc.
+SPDX-License-Identifier: Apache-2.0
+*/
+
+package provider
+
+type DynamicTLSServingCertProvider struct {
+	CertPEM []byte
+	KeyPEM  []byte
+}
+
+func (*DynamicTLSServingCertProvider) Name() string {
+	return "DynamicTLSServingCertProvider"
+}
+
+func (p *DynamicTLSServingCertProvider) CurrentCertKeyContent() (cert []byte, key []byte) {
+	return p.CertPEM, p.KeyPEM
+}

internal/server/server.go

@@ -8,24 +8,19 @@ package server
 
 import (
 	"context"
-	"crypto/tls"
-	"crypto/x509"
-	"crypto/x509/pkix"
-	"encoding/pem"
 	"fmt"
 	"io"
-	"time"
 
 	"github.com/spf13/cobra"
 	genericapiserver "k8s.io/apiserver/pkg/server"
-	"k8s.io/apiserver/pkg/server/dynamiccertificates"
 	genericoptions "k8s.io/apiserver/pkg/server/options"
 	"k8s.io/apiserver/plugin/pkg/authenticator/token/webhook"
 
 	"github.com/suzerain-io/placeholder-name/internal/apiserver"
 	"github.com/suzerain-io/placeholder-name/internal/certauthority"
-	"github.com/suzerain-io/placeholder-name/internal/controller"
+	"github.com/suzerain-io/placeholder-name/internal/controllermanager"
 	"github.com/suzerain-io/placeholder-name/internal/downward"
+	"github.com/suzerain-io/placeholder-name/internal/provider"
 	placeholderv1alpha1 "github.com/suzerain-io/placeholder-name/kubernetes/1.19/api/apis/placeholder/v1alpha1"
 	"github.com/suzerain-io/placeholder-name/pkg/config"
 )
@@ -135,31 +130,19 @@ func (app *App) runServer(ctx context.Context) error {
 	}
 	serverInstallationNamespace := podInfo.Namespace
 
-	// Create a CA.
+	// This cert provider will provide certs to the API server and will
-	aggregatedAPIServerCA, err := certauthority.New(pkix.Name{CommonName: "Placeholder CA"})
+	// be mutated by a controller to keep the certs up to date with what
-	if err != nil {
+	// is stored in a k8s Secret. Therefore it also effectively acting as
-		return fmt.Errorf("could not initialize CA: %w", err)
+	// an in-memory cache of what is stored in the k8s Secret, helping to
-	}
+	// keep incoming requests fast.
-
+	dynamicCertProvider := &provider.DynamicTLSServingCertProvider{}
-	// This string must match the name of the Service declared in the deployment yaml.
-	const serviceName = "placeholder-name-api"
-	// Using the CA from above, create a TLS server cert for the aggregated API server to use.
-	aggregatedAPIServerTLSCert, err := aggregatedAPIServerCA.Issue(
-		pkix.Name{CommonName: serviceName + "." + serverInstallationNamespace + ".svc"},
-		[]string{},
-		24*365*time.Hour,
-	)
-	if err != nil {
-		return fmt.Errorf("could not issue serving certificate: %w", err)
-	}
 
 	// Prepare to start the controllers, but defer actually starting them until the
 	// post start hook of the aggregated API server.
-	startControllersFunc, err := controller.PrepareControllers(
+	startControllersFunc, err := controllermanager.PrepareControllers(
-		ctx,
-		aggregatedAPIServerCA.Bundle(),
 		serverInstallationNamespace,
 		cfg.DiscoveryConfig.URL,
+		dynamicCertProvider,
 	)
 	if err != nil {
 		return fmt.Errorf("could not prepare controllers: %w", err)
@@ -167,7 +150,7 @@ func (app *App) runServer(ctx context.Context) error {
 
 	// Get the aggregated API server config.
 	aggregatedAPIServerConfig, err := getAggregatedAPIServerConfig(
-		aggregatedAPIServerTLSCert,
+		dynamicCertProvider,
 		webhookTokenAuthenticator,
 		k8sClusterCA,
 		startControllersFunc,
@@ -188,25 +171,29 @@ func (app *App) runServer(ctx context.Context) error {
 
 // Create a configuration for the aggregated API server.
 func getAggregatedAPIServerConfig(
-	cert *tls.Certificate,
+	dynamicCertProvider *provider.DynamicTLSServingCertProvider,
 	webhookTokenAuthenticator *webhook.WebhookTokenAuthenticator,
 	ca *certauthority.CA,
 	startControllersPostStartHook func(context.Context),
 ) (*apiserver.Config, error) {
-	provider, err := createStaticCertKeyProvider(cert)
-	if err != nil {
-		return nil, fmt.Errorf("could not create static cert key provider: %w", err)
-	}
-
 	recommendedOptions := genericoptions.NewRecommendedOptions(
 		defaultEtcdPathPrefix,
 		apiserver.Codecs.LegacyCodec(placeholderv1alpha1.SchemeGroupVersion),
 		// TODO we should check to see if all the other default settings are acceptable for us
 	)
 	recommendedOptions.Etcd = nil // turn off etcd storage because we don't need it yet
-	recommendedOptions.SecureServing.ServerCert.GeneratedCert = provider
+	recommendedOptions.SecureServing.ServerCert.GeneratedCert = dynamicCertProvider
 
 	serverConfig := genericapiserver.NewRecommendedConfig(apiserver.Codecs)
+	// Note that among other things, this ApplyTo() function copies
+	// `recommendedOptions.SecureServing.ServerCert.GeneratedCert` into
+	// `serverConfig.SecureServing.Cert` thus making `dynamicCertProvider`
+	// the cert provider for the running server. The provider will be called
+	// by the API machinery periodically. When the provider returns nil certs,
+	// the API server will return "the server is currently unable to
+	// handle the request" error responses for all incoming requests.
+	// If the provider later starts returning certs, then the API server
+	// will use them to handle the incoming requests successfully.
 	if err := recommendedOptions.ApplyTo(serverConfig); err != nil {
 		return nil, err
 	}
@@ -221,27 +208,3 @@ func getAggregatedAPIServerConfig(
 	}
 	return apiServerConfig, nil
 }
-
-func createStaticCertKeyProvider(cert *tls.Certificate) (dynamiccertificates.CertKeyContentProvider, error) {
-	privateKeyDER, err := x509.MarshalPKCS8PrivateKey(cert.PrivateKey)
-	if err != nil {
-		return nil, fmt.Errorf("error marshalling private key: %w", err)
-	}
-	privateKeyPEM := pem.EncodeToMemory(&pem.Block{
-		Type:    "PRIVATE KEY",
-		Headers: nil,
-		Bytes:   privateKeyDER,
-	})
-
-	certChainPEM := make([]byte, 0)
-	for _, certFromChain := range cert.Certificate {
-		certPEMBytes := pem.EncodeToMemory(&pem.Block{
-			Type:    "CERTIFICATE",
-			Headers: nil,
-			Bytes:   certFromChain,
-		})
-		certChainPEM = append(certChainPEM, certPEMBytes...)
-	}
-
-	return dynamiccertificates.NewStaticCertKeyContent("some-name???", certChainPEM, privateKeyPEM)
-}