Merge pull request #43 from suzerain-io/cert_controllers

Refactor cert generation code into controllers

Dockerfile

@@ -1,3 +1,6 @@
+# Copyright 2020 VMware, Inc.
+# SPDX-License-Identifier: Apache-2.0
+
 FROM golang:1.14.7-alpine as build-env
 
 # It is important that these ARG's are defined after the FROM statement

README.md

@@ -1 +1,3 @@
-# placeholder-name
+# placeholder-name
+
+Copyright 2020 VMware, Inc.

deploy/rbac.yaml

@@ -1,5 +1,6 @@
 #@ load("@ytt:data", "data")
 
+#! Give permission to various cluster-scoped objects
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
@@ -28,6 +29,8 @@ roleRef:
   kind: ClusterRole
   name: #@ data.values.app_name + "-aggregated-api-server-cluster-role"
   apiGroup: rbac.authorization.k8s.io
+
+#! Give permission to various objects within the app's own namespace
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
@@ -38,6 +41,9 @@ rules:
   - apiGroups: [""]
     resources: [services]
     verbs: [create, get, list, patch, update, watch]
+  - apiGroups: [""]
+    resources: [secrets]
+    verbs: [create, get, list, patch, update, watch]
   - apiGroups: [crds.placeholder.suzerain-io.github.io]
     resources: [logindiscoveryconfigs]
     verbs: [create, get, list, update, watch]
@@ -55,6 +61,8 @@ roleRef:
   kind: Role
   name: #@ data.values.app_name + "-aggregated-api-server-role"
   apiGroup: rbac.authorization.k8s.io
+
+#! Allow both authenticated and unauthenticated LoginRequests (i.e. allow all requests)
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
@@ -70,7 +78,6 @@ apiVersion: rbac.authorization.k8s.io/v1
 metadata:
   name: #@ data.values.app_name + "-loginrequests-cluster-role-binding"
 subjects:
-  #! both authenticated and unauthenticated requests (i.e. all requests) should be allowed
   - kind: Group
     name: system:authenticated
     apiGroup: rbac.authorization.k8s.io
@@ -81,6 +88,8 @@ roleRef:
   kind: ClusterRole
   name: #@ data.values.app_name + "-loginrequests-cluster-role"
   apiGroup: rbac.authorization.k8s.io
+
+#! Give permissions for subjectaccessreviews, tokenreview that is needed by aggregated api servers
 ---
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
@@ -93,9 +102,10 @@ subjects:
     namespace: #@ data.values.namespace
 roleRef:
   kind: ClusterRole
-  #! give permissions for subjectaccessreviews, tokenreview that is needed by aggregated api servers
   name: system:auth-delegator
   apiGroup: rbac.authorization.k8s.io
+
+#! Give permissions for a special configmap of CA bundles that is needed by aggregated api servers
 ---
 kind: RoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
@@ -108,9 +118,10 @@ subjects:
     namespace: #@ data.values.namespace
 roleRef:
   kind: Role
-  #! give permissions for a special configmap of CA bundles that is needed by aggregated api servers
   name: extension-apiserver-authentication-reader
   apiGroup: rbac.authorization.k8s.io
+
+#! Give permission to list and watch ConfigMaps in kube-public
 ---
 kind: Role
 apiVersion: rbac.authorization.k8s.io/v1

internal/certauthority/certauthority.go

@@ -195,6 +195,16 @@ func toPEM(cert *tls.Certificate, err error) ([]byte, []byte, error) {
 		return nil, nil, err
 	}
 
+	certPEM, keyPEM, err := ToPEM(cert)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return certPEM, keyPEM, nil
+}
+
+// Encode a tls.Certificate into a private key PEM and a cert chain PEM.
+func ToPEM(cert *tls.Certificate) ([]byte, []byte, error) {
 	// Encode the certificate(s) to PEM.
 	certPEMBlocks := make([][]byte, 0, len(cert.Certificate))
 	for _, c := range cert.Certificate {

internal/controller/apicerts/certs_manager.go

@@ -0,0 +1,132 @@
+/*
+Copyright 2020 VMware, Inc.
+SPDX-License-Identifier: Apache-2.0
+*/
+
+package apicerts
+
+import (
+	"crypto/x509/pkix"
+	"fmt"
+	"time"
+
+	corev1 "k8s.io/api/core/v1"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	corev1informers "k8s.io/client-go/informers/core/v1"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/klog/v2"
+	aggregatorclient "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset"
+
+	"github.com/suzerain-io/controller-go"
+	"github.com/suzerain-io/placeholder-name/internal/certauthority"
+	placeholdernamecontroller "github.com/suzerain-io/placeholder-name/internal/controller"
+)
+
+const (
+	//nolint: gosec
+	certsSecretName              = "api-serving-cert"
+	caCertificateSecretKey       = "caCertificate"
+	tlsPrivateKeySecretKey       = "tlsPrivateKey"
+	tlsCertificateChainSecretKey = "tlsCertificateChain"
+)
+
+type certsManagerController struct {
+	namespace        string
+	k8sClient        kubernetes.Interface
+	aggregatorClient aggregatorclient.Interface
+	secretInformer   corev1informers.SecretInformer
+}
+
+func NewCertsManagerController(
+	namespace string,
+	k8sClient kubernetes.Interface,
+	aggregatorClient aggregatorclient.Interface,
+	secretInformer corev1informers.SecretInformer,
+	withInformer placeholdernamecontroller.WithInformerOptionFunc,
+	withInitialEvent placeholdernamecontroller.WithInitialEventOptionFunc,
+) controller.Controller {
+	return controller.New(
+		controller.Config{
+			Name: "certs-manager-controller",
+			Syncer: &certsManagerController{
+				namespace:        namespace,
+				k8sClient:        k8sClient,
+				aggregatorClient: aggregatorClient,
+				secretInformer:   secretInformer,
+			},
+		},
+		withInformer(
+			secretInformer,
+			placeholdernamecontroller.NameAndNamespaceExactMatchFilterFactory(certsSecretName, namespace),
+			controller.InformerOption{},
+		),
+		// Be sure to run once even if the Secret that the informer is watching doesn't exist.
+		withInitialEvent(controller.Key{
+			Namespace: namespace,
+			Name:      certsSecretName,
+		}),
+	)
+}
+
+func (c *certsManagerController) Sync(ctx controller.Context) error {
+	// Try to get the secret from the informer cache.
+	_, err := c.secretInformer.Lister().Secrets(c.namespace).Get(certsSecretName)
+	notFound := k8serrors.IsNotFound(err)
+	if err != nil && !notFound {
+		return fmt.Errorf("failed to get %s/%s secret: %w", c.namespace, certsSecretName, err)
+	}
+	if !notFound {
+		// The secret already exists, so nothing to do.
+		return nil
+	}
+
+	// Create a CA.
+	aggregatedAPIServerCA, err := certauthority.New(pkix.Name{CommonName: "Placeholder CA"})
+	if err != nil {
+		return fmt.Errorf("could not initialize CA: %w", err)
+	}
+
+	// This string must match the name of the Service declared in the deployment yaml.
+	const serviceName = "placeholder-name-api"
+
+	// Using the CA from above, create a TLS server cert for the aggregated API server to use.
+	aggregatedAPIServerTLSCert, err := aggregatedAPIServerCA.Issue(
+		pkix.Name{CommonName: serviceName + "." + c.namespace + ".svc"},
+		[]string{},
+		24*365*time.Hour,
+	)
+	if err != nil {
+		return fmt.Errorf("could not issue serving certificate: %w", err)
+	}
+
+	// Write the CA's public key bundle and the serving certs to a secret.
+	tlsCertChainPEM, tlsPrivateKeyPEM, err := certauthority.ToPEM(aggregatedAPIServerTLSCert)
+	if err != nil {
+		return fmt.Errorf("could not PEM encode serving certificate: %w", err)
+	}
+	secret := corev1.Secret{
+		TypeMeta: metav1.TypeMeta{},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      certsSecretName,
+			Namespace: c.namespace,
+		},
+		StringData: map[string]string{
+			caCertificateSecretKey:       string(aggregatedAPIServerCA.Bundle()),
+			tlsPrivateKeySecretKey:       string(tlsPrivateKeyPEM),
+			tlsCertificateChainSecretKey: string(tlsCertChainPEM),
+		},
+	}
+	_, err = c.k8sClient.CoreV1().Secrets(c.namespace).Create(ctx.Context, &secret, metav1.CreateOptions{})
+	if err != nil {
+		return fmt.Errorf("could not create secret: %w", err)
+	}
+
+	// Update the APIService to give it the new CA bundle.
+	if err := UpdateAPIService(ctx.Context, c.aggregatorClient, aggregatedAPIServerCA.Bundle()); err != nil {
+		return fmt.Errorf("could not update the API service: %w", err)
+	}
+
+	klog.Info("certsManagerController Sync successfully created secret and updated API service")
+	return nil
+}

internal/controller/apicerts/certs_manager_test.go

@@ -0,0 +1,349 @@
+/*
+Copyright 2020 VMware, Inc.
+SPDX-License-Identifier: Apache-2.0
+*/
+
+package apicerts
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/pem"
+	"errors"
+	"testing"
+	"time"
+
+	"github.com/sclevine/spec"
+	"github.com/sclevine/spec/report"
+	"github.com/stretchr/testify/require"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	kubeinformers "k8s.io/client-go/informers"
+	kubernetesfake "k8s.io/client-go/kubernetes/fake"
+	coretesting "k8s.io/client-go/testing"
+	apiregistrationv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1"
+	aggregatorfake "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/fake"
+
+	"github.com/suzerain-io/controller-go"
+	"github.com/suzerain-io/placeholder-name/internal/testutil"
+	placeholderv1alpha1 "github.com/suzerain-io/placeholder-name/kubernetes/1.19/api/apis/placeholder/v1alpha1"
+)
+
+func TestManagerControllerOptions(t *testing.T) {
+	spec.Run(t, "options", func(t *testing.T, when spec.G, it spec.S) {
+		const installedInNamespace = "some-namespace"
+
+		var r *require.Assertions
+		var observableWithInformerOption *testutil.ObservableWithInformerOption
+		var observableWithInitialEventOption *testutil.ObservableWithInitialEventOption
+		var secretsInformerFilter controller.Filter
+
+		it.Before(func() {
+			r = require.New(t)
+			observableWithInformerOption = testutil.NewObservableWithInformerOption()
+			observableWithInitialEventOption = testutil.NewObservableWithInitialEventOption()
+			secretsInformer := kubeinformers.NewSharedInformerFactory(nil, 0).Core().V1().Secrets()
+			_ = NewCertsManagerController(
+				installedInNamespace,
+				nil,
+				nil,
+				secretsInformer,
+				observableWithInformerOption.WithInformer,         // make it possible to observe the behavior of the Filters
+				observableWithInitialEventOption.WithInitialEvent, // make it possible to observe the behavior of the initial event
+			)
+			secretsInformerFilter = observableWithInformerOption.GetFilterForInformer(secretsInformer)
+		})
+
+		when("watching Secret objects", func() {
+			var subject controller.Filter
+			var target, wrongNamespace, wrongName, unrelated *corev1.Secret
+
+			it.Before(func() {
+				subject = secretsInformerFilter
+				target = &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "api-serving-cert", Namespace: installedInNamespace}}
+				wrongNamespace = &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "api-serving-cert", Namespace: "wrong-namespace"}}
+				wrongName = &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "wrong-name", Namespace: installedInNamespace}}
+				unrelated = &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "wrong-name", Namespace: "wrong-namespace"}}
+			})
+
+			when("the target Secret changes", func() {
+				it("returns true to trigger the sync method", func() {
+					r.True(subject.Add(target))
+					r.True(subject.Update(target, unrelated))
+					r.True(subject.Update(unrelated, target))
+					r.True(subject.Delete(target))
+				})
+			})
+
+			when("a Secret from another namespace changes", func() {
+				it("returns false to avoid triggering the sync method", func() {
+					r.False(subject.Add(wrongNamespace))
+					r.False(subject.Update(wrongNamespace, unrelated))
+					r.False(subject.Update(unrelated, wrongNamespace))
+					r.False(subject.Delete(wrongNamespace))
+				})
+			})
+
+			when("a Secret with a different name changes", func() {
+				it("returns false to avoid triggering the sync method", func() {
+					r.False(subject.Add(wrongName))
+					r.False(subject.Update(wrongName, unrelated))
+					r.False(subject.Update(unrelated, wrongName))
+					r.False(subject.Delete(wrongName))
+				})
+			})
+
+			when("a Secret with a different name and a different namespace changes", func() {
+				it("returns false to avoid triggering the sync method", func() {
+					r.False(subject.Add(unrelated))
+					r.False(subject.Update(unrelated, unrelated))
+					r.False(subject.Delete(unrelated))
+				})
+			})
+		})
+
+		when("starting up", func() {
+			it("asks for an initial event because the Secret may not exist yet and it needs to run anyway", func() {
+				r.Equal(controller.Key{
+					Namespace: installedInNamespace,
+					Name:      "api-serving-cert",
+				}, observableWithInitialEventOption.GetInitialEventKey())
+			})
+		})
+	}, spec.Parallel(), spec.Report(report.Terminal{}))
+}
+
+func TestManagerControllerSync(t *testing.T) {
+	spec.Run(t, "Sync", func(t *testing.T, when spec.G, it spec.S) {
+		const installedInNamespace = "some-namespace"
+
+		var r *require.Assertions
+
+		var subject controller.Controller
+		var kubeAPIClient *kubernetesfake.Clientset
+		var aggregatorAPIClient *aggregatorfake.Clientset
+		var kubeInformerClient *kubernetesfake.Clientset
+		var kubeInformers kubeinformers.SharedInformerFactory
+		var timeoutContext context.Context
+		var timeoutContextCancel context.CancelFunc
+		var syncContext *controller.Context
+
+		// Defer starting the informers until the last possible moment so that the
+		// nested Before's can keep adding things to the informer caches.
+		var startInformersAndController = func() {
+			// Set this at the last second to allow for injection of server override.
+			subject = NewCertsManagerController(
+				installedInNamespace,
+				kubeAPIClient,
+				aggregatorAPIClient,
+				kubeInformers.Core().V1().Secrets(),
+				controller.WithInformer,
+				controller.WithInitialEvent,
+			)
+
+			// Set this at the last second to support calling subject.Name().
+			syncContext = &controller.Context{
+				Context: timeoutContext,
+				Name:    subject.Name(),
+				Key: controller.Key{
+					Namespace: installedInNamespace,
+					Name:      "api-serving-cert",
+				},
+			}
+
+			// Must start informers before calling TestRunSynchronously()
+			kubeInformers.Start(timeoutContext.Done())
+			controller.TestRunSynchronously(t, subject)
+		}
+
+		it.Before(func() {
+			r = require.New(t)
+
+			timeoutContext, timeoutContextCancel = context.WithTimeout(context.Background(), time.Second*3)
+
+			kubeInformerClient = kubernetesfake.NewSimpleClientset()
+			kubeInformers = kubeinformers.NewSharedInformerFactory(kubeInformerClient, 0)
+			kubeAPIClient = kubernetesfake.NewSimpleClientset()
+			aggregatorAPIClient = aggregatorfake.NewSimpleClientset()
+		})
+
+		it.After(func() {
+			timeoutContextCancel()
+		})
+
+		when("there is not yet an api-serving-cert Secret in the installation namespace or it was deleted", func() {
+			it.Before(func() {
+				unrelatedSecret := &corev1.Secret{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      "some other secret",
+						Namespace: installedInNamespace,
+					},
+				}
+				err := kubeInformerClient.Tracker().Add(unrelatedSecret)
+				r.NoError(err)
+			})
+
+			when("the APIService exists", func() {
+				it.Before(func() {
+					apiService := &apiregistrationv1.APIService{
+						ObjectMeta: metav1.ObjectMeta{
+							Name: placeholderv1alpha1.SchemeGroupVersion.Version + "." + placeholderv1alpha1.GroupName,
+						},
+						Spec: apiregistrationv1.APIServiceSpec{
+							CABundle:        nil,
+							VersionPriority: 1234,
+						},
+					}
+					err := aggregatorAPIClient.Tracker().Add(apiService)
+					r.NoError(err)
+				})
+
+				it("creates the api-serving-cert Secret and updates the APIService's ca bundle", func() {
+					startInformersAndController()
+					err := controller.TestSync(t, subject, *syncContext)
+					r.NoError(err)
+
+					// Check all the relevant fields from the create Secret action
+					r.Len(kubeAPIClient.Actions(), 1)
+					actualAction := kubeAPIClient.Actions()[0].(coretesting.CreateActionImpl)
+					r.Equal(schema.GroupVersionResource{Group: "", Version: "v1", Resource: "secrets"}, actualAction.GetResource())
+					r.Equal(installedInNamespace, actualAction.GetNamespace())
+					actualSecret := actualAction.GetObject().(*corev1.Secret)
+					r.Equal("api-serving-cert", actualSecret.Name)
+					r.Equal(installedInNamespace, actualSecret.Namespace)
+					actualCACert := actualSecret.StringData["caCertificate"]
+					actualPrivateKey := actualSecret.StringData["tlsPrivateKey"]
+					actualCertChain := actualSecret.StringData["tlsCertificateChain"]
+					r.NotEmpty(actualCACert)
+					r.NotEmpty(actualPrivateKey)
+					r.NotEmpty(actualCertChain)
+
+					// Validate the created cert using the CA, and also validate the cert's hostname
+					roots := x509.NewCertPool()
+					ok := roots.AppendCertsFromPEM([]byte(actualCACert))
+					r.True(ok)
+					block, _ := pem.Decode([]byte(actualCertChain))
+					r.NotNil(block)
+					parsedCert, err := x509.ParseCertificate(block.Bytes)
+					r.NoError(err)
+					opts := x509.VerifyOptions{
+						DNSName: "placeholder-name-api." + installedInNamespace + ".svc",
+						Roots:   roots,
+					}
+					_, err = parsedCert.Verify(opts)
+					r.NoError(err)
+
+					// Check the created cert's validity bounds
+					r.WithinDuration(time.Now(), parsedCert.NotBefore, time.Minute*2)
+					r.WithinDuration(time.Now().Add(24*365*time.Hour), parsedCert.NotAfter, time.Minute*2)
+
+					// Check that the private key and cert chain match
+					_, err = tls.X509KeyPair([]byte(actualCertChain), []byte(actualPrivateKey))
+					r.NoError(err)
+
+					// Make sure we updated the APIService caBundle and left it otherwise unchanged
+					r.Len(aggregatorAPIClient.Actions(), 2)
+					r.Equal("get", aggregatorAPIClient.Actions()[0].GetVerb())
+					expectedAPIServiceName := placeholderv1alpha1.SchemeGroupVersion.Version + "." + placeholderv1alpha1.GroupName
+					expectedUpdateAction := coretesting.NewUpdateAction(
+						schema.GroupVersionResource{
+							Group:    apiregistrationv1.GroupName,
+							Version:  "v1",
+							Resource: "apiservices",
+						},
+						"",
+						&apiregistrationv1.APIService{
+							ObjectMeta: metav1.ObjectMeta{
+								Name:      expectedAPIServiceName,
+								Namespace: "",
+							},
+							Spec: apiregistrationv1.APIServiceSpec{
+								VersionPriority: 1234, // only the CABundle is updated, this other field is left unchanged
+								CABundle:        []byte(actualCACert),
+							},
+						},
+					)
+					r.Equal(expectedUpdateAction, aggregatorAPIClient.Actions()[1])
+				})
+
+				when("updating the APIService fails", func() {
+					it.Before(func() {
+						aggregatorAPIClient.PrependReactor(
+							"update",
+							"apiservices",
+							func(_ coretesting.Action) (bool, runtime.Object, error) {
+								return true, nil, errors.New("update failed")
+							},
+						)
+					})
+
+					it("returns the update error", func() {
+						startInformersAndController()
+						err := controller.TestSync(t, subject, *syncContext)
+						r.EqualError(err, "could not update the API service: could not update API service: update failed")
+					})
+				})
+			})
+
+			when("the APIService does not exist", func() {
+				it.Before(func() {
+					unrelatedAPIService := &apiregistrationv1.APIService{
+						ObjectMeta: metav1.ObjectMeta{Name: "some other api service"},
+						Spec:       apiregistrationv1.APIServiceSpec{},
+					}
+					err := aggregatorAPIClient.Tracker().Add(unrelatedAPIService)
+					r.NoError(err)
+				})
+
+				it("returns an error", func() {
+					startInformersAndController()
+					err := controller.TestSync(t, subject, *syncContext)
+					r.Error(err)
+					r.Regexp("could not get existing version of API service: .* not found", err.Error())
+				})
+			})
+
+			when("creating the Secret fails", func() {
+				it.Before(func() {
+					kubeAPIClient.PrependReactor(
+						"create",
+						"secrets",
+						func(_ coretesting.Action) (bool, runtime.Object, error) {
+							return true, nil, errors.New("create failed")
+						},
+					)
+				})
+
+				it("returns the create error", func() {
+					startInformersAndController()
+					err := controller.TestSync(t, subject, *syncContext)
+					r.EqualError(err, "could not create secret: create failed")
+				})
+			})
+		})
+
+		when("there is an api-serving-cert Secret already in the installation namespace", func() {
+			it.Before(func() {
+				apiServingCertSecret := &corev1.Secret{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      "api-serving-cert",
+						Namespace: installedInNamespace,
+					},
+				}
+				err := kubeInformerClient.Tracker().Add(apiServingCertSecret)
+				r.NoError(err)
+			})
+
+			it("does not need to make any API calls with its API clients", func() {
+				startInformersAndController()
+				err := controller.TestSync(t, subject, *syncContext)
+				r.NoError(err)
+				r.Empty(kubeAPIClient.Actions())
+				r.Empty(aggregatorAPIClient.Actions())
+			})
+		})
+	}, spec.Parallel(), spec.Report(report.Terminal{}))
+}

internal/controller/apicerts/certs_observer.go

@@ -0,0 +1,67 @@
+/*
+Copyright 2020 VMware, Inc.
+SPDX-License-Identifier: Apache-2.0
+*/
+
+package apicerts
+
+import (
+	"fmt"
+
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	corev1informers "k8s.io/client-go/informers/core/v1"
+	"k8s.io/klog/v2"
+
+	"github.com/suzerain-io/controller-go"
+	placeholdernamecontroller "github.com/suzerain-io/placeholder-name/internal/controller"
+	"github.com/suzerain-io/placeholder-name/internal/provider"
+)
+
+type certsObserverController struct {
+	namespace           string
+	dynamicCertProvider provider.DynamicTLSServingCertProvider
+	secretInformer      corev1informers.SecretInformer
+}
+
+func NewCertsObserverController(
+	namespace string,
+	dynamicCertProvider provider.DynamicTLSServingCertProvider,
+	secretInformer corev1informers.SecretInformer,
+	withInformer placeholdernamecontroller.WithInformerOptionFunc,
+) controller.Controller {
+	return controller.New(
+		controller.Config{
+			Name: "certs-observer-controller",
+			Syncer: &certsObserverController{
+				namespace:           namespace,
+				dynamicCertProvider: dynamicCertProvider,
+				secretInformer:      secretInformer,
+			},
+		},
+		withInformer(
+			secretInformer,
+			placeholdernamecontroller.NameAndNamespaceExactMatchFilterFactory(certsSecretName, namespace),
+			controller.InformerOption{},
+		),
+	)
+}
+
+func (c *certsObserverController) Sync(_ controller.Context) error {
+	// Try to get the secret from the informer cache.
+	certSecret, err := c.secretInformer.Lister().Secrets(c.namespace).Get(certsSecretName)
+	notFound := k8serrors.IsNotFound(err)
+	if err != nil && !notFound {
+		return fmt.Errorf("failed to get %s/%s secret: %w", c.namespace, certsSecretName, err)
+	}
+	if notFound {
+		klog.Info("certsObserverController Sync() found that the secret does not exist yet or was deleted")
+		// The secret does not exist yet or was deleted.
+		c.dynamicCertProvider.Set(nil, nil)
+		return nil
+	}
+
+	// Mutate the in-memory cert provider to update with the latest cert values.
+	c.dynamicCertProvider.Set(certSecret.Data[tlsCertificateChainSecretKey], certSecret.Data[tlsPrivateKeySecretKey])
+	klog.Info("certsObserverController Sync updated certs in the dynamic cert provider")
+	return nil
+}

internal/controller/apicerts/certs_observer_test.go

@@ -0,0 +1,232 @@
+/*
+Copyright 2020 VMware, Inc.
+SPDX-License-Identifier: Apache-2.0
+*/
+
+package apicerts
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/sclevine/spec"
+	"github.com/sclevine/spec/report"
+	"github.com/stretchr/testify/require"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	kubeinformers "k8s.io/client-go/informers"
+	kubernetesfake "k8s.io/client-go/kubernetes/fake"
+
+	"github.com/suzerain-io/controller-go"
+	"github.com/suzerain-io/placeholder-name/internal/provider"
+	"github.com/suzerain-io/placeholder-name/internal/testutil"
+)
+
+func TestObserverControllerInformerFilters(t *testing.T) {
+	spec.Run(t, "informer filters", func(t *testing.T, when spec.G, it spec.S) {
+		const installedInNamespace = "some-namespace"
+
+		var r *require.Assertions
+		var observableWithInformerOption *testutil.ObservableWithInformerOption
+		var secretsInformerFilter controller.Filter
+
+		it.Before(func() {
+			r = require.New(t)
+			observableWithInformerOption = testutil.NewObservableWithInformerOption()
+			secretsInformer := kubeinformers.NewSharedInformerFactory(nil, 0).Core().V1().Secrets()
+			_ = NewCertsObserverController(
+				installedInNamespace,
+				nil,
+				secretsInformer,
+				observableWithInformerOption.WithInformer, // make it possible to observe the behavior of the Filters
+			)
+			secretsInformerFilter = observableWithInformerOption.GetFilterForInformer(secretsInformer)
+		})
+
+		when("watching Secret objects", func() {
+			var subject controller.Filter
+			var target, wrongNamespace, wrongName, unrelated *corev1.Secret
+
+			it.Before(func() {
+				subject = secretsInformerFilter
+				target = &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "api-serving-cert", Namespace: installedInNamespace}}
+				wrongNamespace = &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "api-serving-cert", Namespace: "wrong-namespace"}}
+				wrongName = &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "wrong-name", Namespace: installedInNamespace}}
+				unrelated = &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "wrong-name", Namespace: "wrong-namespace"}}
+			})
+
+			when("the target Secret changes", func() {
+				it("returns true to trigger the sync method", func() {
+					r.True(subject.Add(target))
+					r.True(subject.Update(target, unrelated))
+					r.True(subject.Update(unrelated, target))
+					r.True(subject.Delete(target))
+				})
+			})
+
+			when("a Secret from another namespace changes", func() {
+				it("returns false to avoid triggering the sync method", func() {
+					r.False(subject.Add(wrongNamespace))
+					r.False(subject.Update(wrongNamespace, unrelated))
+					r.False(subject.Update(unrelated, wrongNamespace))
+					r.False(subject.Delete(wrongNamespace))
+				})
+			})
+
+			when("a Secret with a different name changes", func() {
+				it("returns false to avoid triggering the sync method", func() {
+					r.False(subject.Add(wrongName))
+					r.False(subject.Update(wrongName, unrelated))
+					r.False(subject.Update(unrelated, wrongName))
+					r.False(subject.Delete(wrongName))
+				})
+			})
+
+			when("a Secret with a different name and a different namespace changes", func() {
+				it("returns false to avoid triggering the sync method", func() {
+					r.False(subject.Add(unrelated))
+					r.False(subject.Update(unrelated, unrelated))
+					r.False(subject.Delete(unrelated))
+				})
+			})
+		})
+	}, spec.Parallel(), spec.Report(report.Terminal{}))
+}
+
+func TestObserverControllerSync(t *testing.T) {
+	spec.Run(t, "Sync", func(t *testing.T, when spec.G, it spec.S) {
+		const installedInNamespace = "some-namespace"
+
+		var r *require.Assertions
+
+		var subject controller.Controller
+		var kubeInformerClient *kubernetesfake.Clientset
+		var kubeInformers kubeinformers.SharedInformerFactory
+		var timeoutContext context.Context
+		var timeoutContextCancel context.CancelFunc
+		var syncContext *controller.Context
+		var dynamicCertProvider provider.DynamicTLSServingCertProvider
+
+		// Defer starting the informers until the last possible moment so that the
+		// nested Before's can keep adding things to the informer caches.
+		var startInformersAndController = func() {
+			// Set this at the last second to allow for injection of server override.
+			subject = NewCertsObserverController(
+				installedInNamespace,
+				dynamicCertProvider,
+				kubeInformers.Core().V1().Secrets(),
+				controller.WithInformer,
+			)
+
+			// Set this at the last second to support calling subject.Name().
+			syncContext = &controller.Context{
+				Context: timeoutContext,
+				Name:    subject.Name(),
+				Key: controller.Key{
+					Namespace: installedInNamespace,
+					Name:      "api-serving-cert",
+				},
+			}
+
+			// Must start informers before calling TestRunSynchronously()
+			kubeInformers.Start(timeoutContext.Done())
+			controller.TestRunSynchronously(t, subject)
+		}
+
+		it.Before(func() {
+			r = require.New(t)
+
+			timeoutContext, timeoutContextCancel = context.WithTimeout(context.Background(), time.Second*3)
+
+			kubeInformerClient = kubernetesfake.NewSimpleClientset()
+			kubeInformers = kubeinformers.NewSharedInformerFactory(kubeInformerClient, 0)
+			dynamicCertProvider = provider.NewDynamicTLSServingCertProvider()
+		})
+
+		it.After(func() {
+			timeoutContextCancel()
+		})
+
+		when("there is not yet an api-serving-cert Secret in the installation namespace or it was deleted", func() {
+			it.Before(func() {
+				unrelatedSecret := &corev1.Secret{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      "some other secret",
+						Namespace: installedInNamespace,
+					},
+				}
+				err := kubeInformerClient.Tracker().Add(unrelatedSecret)
+				r.NoError(err)
+
+				dynamicCertProvider.Set([]byte("some cert"), []byte("some private key"))
+			})
+
+			it("sets the dynamicCertProvider's cert and key to nil", func() {
+				startInformersAndController()
+				err := controller.TestSync(t, subject, *syncContext)
+				r.NoError(err)
+
+				actualCertChain, actualKey := dynamicCertProvider.CurrentCertKeyContent()
+				r.Nil(actualCertChain)
+				r.Nil(actualKey)
+			})
+		})
+
+		when("there is an api-serving-cert Secret with the expected keys already in the installation namespace", func() {
+			it.Before(func() {
+				apiServingCertSecret := &corev1.Secret{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      "api-serving-cert",
+						Namespace: installedInNamespace,
+					},
+					Data: map[string][]byte{
+						"caCertificate":       []byte("fake cert"),
+						"tlsPrivateKey":       []byte("fake private key"),
+						"tlsCertificateChain": []byte("fake cert chain"),
+					},
+				}
+				err := kubeInformerClient.Tracker().Add(apiServingCertSecret)
+				r.NoError(err)
+
+				dynamicCertProvider.Set(nil, nil)
+			})
+
+			it("updates the dynamicCertProvider's cert and key", func() {
+				startInformersAndController()
+				err := controller.TestSync(t, subject, *syncContext)
+				r.NoError(err)
+
+				actualCertChain, actualKey := dynamicCertProvider.CurrentCertKeyContent()
+				r.Equal("fake cert chain", string(actualCertChain))
+				r.Equal("fake private key", string(actualKey))
+			})
+		})
+
+		when("the api-serving-cert Secret exists but is missing the expected keys", func() {
+			it.Before(func() {
+				apiServingCertSecret := &corev1.Secret{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      "api-serving-cert",
+						Namespace: installedInNamespace,
+					},
+					Data: map[string][]byte{},
+				}
+				err := kubeInformerClient.Tracker().Add(apiServingCertSecret)
+				r.NoError(err)
+
+				dynamicCertProvider.Set(nil, nil)
+			})
+
+			it("set the missing values in the dynamicCertProvider as nil", func() {
+				startInformersAndController()
+				err := controller.TestSync(t, subject, *syncContext)
+				r.NoError(err)
+
+				actualCertChain, actualKey := dynamicCertProvider.CurrentCertKeyContent()
+				r.Nil(actualCertChain)
+				r.Nil(actualKey)
+			})
+		})
+	}, spec.Parallel(), spec.Report(report.Terminal{}))
+}

internal/controller/apicerts/update_api_service.go

@@ -3,8 +3,7 @@ Copyright 2020 VMware, Inc.
 SPDX-License-Identifier: Apache-2.0
 */
 
-// Package autoregistration updates the pre-registered APIService.
-package autoregistration
+package apicerts
 
 import (
 	"context"
@@ -12,14 +11,14 @@ import (
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/util/retry"
-	aggregatationv1client "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset"
+	aggregatorclient "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset"
 
 	placeholderv1alpha1 "github.com/suzerain-io/placeholder-name/kubernetes/1.19/api/apis/placeholder/v1alpha1"
 )
 
 // UpdateAPIService updates the APIService's CA bundle.
-func UpdateAPIService(ctx context.Context, aggregationV1 aggregatationv1client.Interface, aggregatedAPIServerCA []byte) error {
-	apiServices := aggregationV1.ApiregistrationV1().APIServices()
+func UpdateAPIService(ctx context.Context, aggregatorClient aggregatorclient.Interface, aggregatedAPIServerCA []byte) error {
+	apiServices := aggregatorClient.ApiregistrationV1().APIServices()
 	apiServiceName := placeholderv1alpha1.SchemeGroupVersion.Version + "." + placeholderv1alpha1.GroupName
 
 	if err := retry.RetryOnConflict(retry.DefaultRetry, func() error {

internal/controller/apicerts/update_api_service_test.go

@@ -3,7 +3,7 @@ Copyright 2020 VMware, Inc.
 SPDX-License-Identifier: Apache-2.0
 */
 
-package autoregistration
+package apicerts
 
 import (
 	"context"
@@ -17,7 +17,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	kubetesting "k8s.io/client-go/testing"
 	apiregistrationv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1"
-	aggregationv1fake "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/fake"
+	aggregatorv1fake "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/fake"
 )
 
 func TestUpdateAPIService(t *testing.T) {
@@ -25,14 +25,14 @@ func TestUpdateAPIService(t *testing.T) {
 
 	tests := []struct {
 		name        string
-		mocks       func(*aggregationv1fake.Clientset)
+		mocks       func(*aggregatorv1fake.Clientset)
 		caInput     []byte
 		wantObjects []apiregistrationv1.APIService
 		wantErr     string
 	}{
 		{
 			name: "happy path update when the pre-existing APIService did not already have a CA bundle",
-			mocks: func(c *aggregationv1fake.Clientset) {
+			mocks: func(c *aggregatorv1fake.Clientset) {
 				_ = c.Tracker().Add(&apiregistrationv1.APIService{
 					ObjectMeta: metav1.ObjectMeta{Name: apiServiceName},
 					Spec: apiregistrationv1.APIServiceSpec{
@@ -52,7 +52,7 @@ func TestUpdateAPIService(t *testing.T) {
 		},
 		{
 			name: "happy path update when the pre-existing APIService already had a CA bundle",
-			mocks: func(c *aggregationv1fake.Clientset) {
+			mocks: func(c *aggregatorv1fake.Clientset) {
 				_ = c.Tracker().Add(&apiregistrationv1.APIService{
 					ObjectMeta: metav1.ObjectMeta{Name: apiServiceName},
 					Spec: apiregistrationv1.APIServiceSpec{
@@ -72,7 +72,7 @@ func TestUpdateAPIService(t *testing.T) {
 		},
 		{
 			name: "error on update",
-			mocks: func(c *aggregationv1fake.Clientset) {
+			mocks: func(c *aggregatorv1fake.Clientset) {
 				_ = c.Tracker().Add(&apiregistrationv1.APIService{
 					ObjectMeta: metav1.ObjectMeta{Name: apiServiceName},
 					Spec:       apiregistrationv1.APIServiceSpec{},
@@ -85,7 +85,7 @@ func TestUpdateAPIService(t *testing.T) {
 		},
 		{
 			name: "error on get",
-			mocks: func(c *aggregationv1fake.Clientset) {
+			mocks: func(c *aggregatorv1fake.Clientset) {
 				_ = c.Tracker().Add(&apiregistrationv1.APIService{
 					ObjectMeta: metav1.ObjectMeta{Name: apiServiceName},
 					Spec:       apiregistrationv1.APIServiceSpec{},
@@ -99,7 +99,7 @@ func TestUpdateAPIService(t *testing.T) {
 		},
 		{
 			name: "conflict error on update, followed by successful retry",
-			mocks: func(c *aggregationv1fake.Clientset) {
+			mocks: func(c *aggregatorv1fake.Clientset) {
 				_ = c.Tracker().Add(&apiregistrationv1.APIService{
 					ObjectMeta: metav1.ObjectMeta{Name: apiServiceName},
 					Spec: apiregistrationv1.APIServiceSpec{
@@ -148,7 +148,7 @@ func TestUpdateAPIService(t *testing.T) {
 		t.Run(tt.name, func(t *testing.T) {
 			ctx := context.Background()
 
-			client := aggregationv1fake.NewSimpleClientset()
+			client := aggregatorv1fake.NewSimpleClientset()
 			if tt.mocks != nil {
 				tt.mocks(client)
 			}

internal/controller/logindiscovery/publisher.go

@@ -17,6 +17,7 @@ import (
 	"k8s.io/klog/v2"
 
 	"github.com/suzerain-io/controller-go"
+	placeholdernamecontroller "github.com/suzerain-io/placeholder-name/internal/controller"
 	crdsplaceholderv1alpha1 "github.com/suzerain-io/placeholder-name/kubernetes/1.19/api/apis/crdsplaceholder/v1alpha1"
 	placeholderclientset "github.com/suzerain-io/placeholder-name/kubernetes/1.19/client-go/clientset/versioned"
 	crdsplaceholderv1alpha1informers "github.com/suzerain-io/placeholder-name/kubernetes/1.19/client-go/informers/externalversions/crdsplaceholder/v1alpha1"
@@ -31,25 +32,6 @@ const (
 	configName = "placeholder-name-config"
 )
 
-func nameAndNamespaceExactMatchFilterFactory(name, namespace string) controller.FilterFuncs {
-	objMatchesFunc := func(obj metav1.Object) bool {
-		return obj.GetName() == name && obj.GetNamespace() == namespace
-	}
-	return controller.FilterFuncs{
-		AddFunc: objMatchesFunc,
-		UpdateFunc: func(oldObj, newObj metav1.Object) bool {
-			return objMatchesFunc(oldObj) || objMatchesFunc(newObj)
-		},
-		DeleteFunc: objMatchesFunc,
-	}
-}
-
-// Same signature as controller.WithInformer().
-type withInformerOptionFunc func(
-	getter controller.InformerGetter,
-	filter controller.Filter,
-	opt controller.InformerOption) controller.Option
-
 type publisherController struct {
 	namespace                    string
 	serverOverride               *string
@@ -64,7 +46,7 @@ func NewPublisherController(
 	placeholderClient placeholderclientset.Interface,
 	configMapInformer corev1informers.ConfigMapInformer,
 	loginDiscoveryConfigInformer crdsplaceholderv1alpha1informers.LoginDiscoveryConfigInformer,
-	withInformer withInformerOptionFunc,
+	withInformer placeholdernamecontroller.WithInformerOptionFunc,
 ) controller.Controller {
 	return controller.New(
 		controller.Config{
@@ -79,12 +61,12 @@ func NewPublisherController(
 		},
 		withInformer(
 			configMapInformer,
-			nameAndNamespaceExactMatchFilterFactory(clusterInfoName, ClusterInfoNamespace),
+			placeholdernamecontroller.NameAndNamespaceExactMatchFilterFactory(clusterInfoName, ClusterInfoNamespace),
 			controller.InformerOption{},
 		),
 		withInformer(
 			loginDiscoveryConfigInformer,
-			nameAndNamespaceExactMatchFilterFactory(configName, namespace),
+			placeholdernamecontroller.NameAndNamespaceExactMatchFilterFactory(configName, namespace),
 			controller.InformerOption{},
 		),
 	)

internal/controller/logindiscovery/publisher_test.go

@@ -24,41 +24,24 @@ import (
 	coretesting "k8s.io/client-go/testing"
 
 	"github.com/suzerain-io/controller-go"
+	"github.com/suzerain-io/placeholder-name/internal/testutil"
 	crdsplaceholderv1alpha1 "github.com/suzerain-io/placeholder-name/kubernetes/1.19/api/apis/crdsplaceholder/v1alpha1"
 	placeholderfake "github.com/suzerain-io/placeholder-name/kubernetes/1.19/client-go/clientset/versioned/fake"
 	placeholderinformers "github.com/suzerain-io/placeholder-name/kubernetes/1.19/client-go/informers/externalversions"
 )
 
-type ObservableWithInformerOption struct {
-	InformerToFilterMap map[controller.InformerGetter]controller.Filter
-}
-
-func NewObservableWithInformerOption() *ObservableWithInformerOption {
-	return &ObservableWithInformerOption{
-		InformerToFilterMap: make(map[controller.InformerGetter]controller.Filter),
-	}
-}
-
-func (owi *ObservableWithInformerOption) WithInformer(
-	getter controller.InformerGetter,
-	filter controller.Filter,
-	opt controller.InformerOption) controller.Option {
-	owi.InformerToFilterMap[getter] = filter
-	return controller.WithInformer(getter, filter, opt)
-}
-
 func TestInformerFilters(t *testing.T) {
 	spec.Run(t, "informer filters", func(t *testing.T, when spec.G, it spec.S) {
 		const installedInNamespace = "some-namespace"
 
 		var r *require.Assertions
-		var observableWithInformerOption *ObservableWithInformerOption
+		var observableWithInformerOption *testutil.ObservableWithInformerOption
 		var configMapInformerFilter controller.Filter
 		var loginDiscoveryConfigInformerFilter controller.Filter
 
 		it.Before(func() {
 			r = require.New(t)
-			observableWithInformerOption = NewObservableWithInformerOption()
+			observableWithInformerOption = testutil.NewObservableWithInformerOption()
 			configMapInformer := kubeinformers.NewSharedInformerFactory(nil, 0).Core().V1().ConfigMaps()
 			loginDiscoveryConfigInformer := placeholderinformers.NewSharedInformerFactory(nil, 0).Crds().V1alpha1().LoginDiscoveryConfigs()
 			_ = NewPublisherController(
@@ -69,8 +52,8 @@ func TestInformerFilters(t *testing.T) {
 				loginDiscoveryConfigInformer,
 				observableWithInformerOption.WithInformer, // make it possible to observe the behavior of the Filters
 			)
-			configMapInformerFilter = observableWithInformerOption.InformerToFilterMap[configMapInformer]
-			loginDiscoveryConfigInformerFilter = observableWithInformerOption.InformerToFilterMap[loginDiscoveryConfigInformer]
+			configMapInformerFilter = observableWithInformerOption.GetFilterForInformer(configMapInformer)
+			loginDiscoveryConfigInformerFilter = observableWithInformerOption.GetFilterForInformer(loginDiscoveryConfigInformer)
 		})
 
 		when("watching ConfigMap objects", func() {

internal/controller/utils.go

@@ -0,0 +1,34 @@
+/*
+Copyright 2020 VMware, Inc.
+SPDX-License-Identifier: Apache-2.0
+*/
+
+package controller
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/suzerain-io/controller-go"
+)
+
+func NameAndNamespaceExactMatchFilterFactory(name, namespace string) controller.FilterFuncs {
+	objMatchesFunc := func(obj metav1.Object) bool {
+		return obj.GetName() == name && obj.GetNamespace() == namespace
+	}
+	return controller.FilterFuncs{
+		AddFunc: objMatchesFunc,
+		UpdateFunc: func(oldObj, newObj metav1.Object) bool {
+			return objMatchesFunc(oldObj) || objMatchesFunc(newObj)
+		},
+		DeleteFunc: objMatchesFunc,
+	}
+}
+
+// Same signature as controller.WithInformer().
+type WithInformerOptionFunc func(
+	getter controller.InformerGetter,
+	filter controller.Filter,
+	opt controller.InformerOption) controller.Option
+
+// Same signature as controller.WithInitialEvent().
+type WithInitialEventOptionFunc func(key controller.Key) controller.Option

internal/controllermanager/prepare_controllers.go

@@ -0,0 +1,169 @@
+/*
+Copyright 2020 VMware, Inc.
+SPDX-License-Identifier: Apache-2.0
+*/
+
+package controllermanager
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"k8s.io/apimachinery/pkg/runtime"
+	k8sinformers "k8s.io/client-go/informers"
+	"k8s.io/client-go/kubernetes"
+	restclient "k8s.io/client-go/rest"
+	aggregatorclient "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset"
+
+	"github.com/suzerain-io/controller-go"
+	"github.com/suzerain-io/placeholder-name/internal/controller/apicerts"
+	"github.com/suzerain-io/placeholder-name/internal/controller/logindiscovery"
+	"github.com/suzerain-io/placeholder-name/internal/provider"
+	placeholderclientset "github.com/suzerain-io/placeholder-name/kubernetes/1.19/client-go/clientset/versioned"
+	placeholderinformers "github.com/suzerain-io/placeholder-name/kubernetes/1.19/client-go/informers/externalversions"
+)
+
+const (
+	singletonWorker       = 1
+	defaultResyncInterval = 3 * time.Minute
+)
+
+// Prepare the controllers and their informers and return a function that will start them when called.
+func PrepareControllers(
+	serverInstallationNamespace string,
+	discoveryURLOverride *string,
+	dynamicCertProvider provider.DynamicTLSServingCertProvider,
+) (func(ctx context.Context), error) {
+	// Create k8s clients.
+	k8sClient, aggregatorClient, placeholderClient, err := createClients()
+	if err != nil {
+		return nil, fmt.Errorf("could not create clients for the controllers: %w", err)
+	}
+
+	// Create informers. Don't forget to make sure they get started in the function returned below.
+	kubePublicNamespaceK8sInformers, installationNamespaceK8sInformers, installationNamespacePlaceholderInformers :=
+		createInformers(serverInstallationNamespace, k8sClient, placeholderClient)
+
+	// Create controller manager.
+	controllerManager := controller.
+		NewManager().
+		WithController(
+			logindiscovery.NewPublisherController(
+				serverInstallationNamespace,
+				discoveryURLOverride,
+				placeholderClient,
+				kubePublicNamespaceK8sInformers.Core().V1().ConfigMaps(),
+				installationNamespacePlaceholderInformers.Crds().V1alpha1().LoginDiscoveryConfigs(),
+				controller.WithInformer,
+			),
+			singletonWorker,
+		).
+		WithController(
+			apicerts.NewCertsManagerController(
+				serverInstallationNamespace,
+				k8sClient,
+				aggregatorClient,
+				installationNamespaceK8sInformers.Core().V1().Secrets(),
+				controller.WithInformer,
+				controller.WithInitialEvent,
+			),
+			singletonWorker,
+		).
+		WithController(
+			apicerts.NewCertsObserverController(
+				serverInstallationNamespace,
+				dynamicCertProvider,
+				installationNamespaceK8sInformers.Core().V1().Secrets(),
+				controller.WithInformer,
+			),
+			singletonWorker,
+		)
+
+	// Return a function which starts the informers and controllers.
+	return func(ctx context.Context) {
+		kubePublicNamespaceK8sInformers.Start(ctx.Done())
+		installationNamespaceK8sInformers.Start(ctx.Done())
+		installationNamespacePlaceholderInformers.Start(ctx.Done())
+
+		go controllerManager.Start(ctx)
+	}, nil
+}
+
+// Create the k8s clients that will be used by the controllers.
+func createClients() (
+	k8sClient *kubernetes.Clientset,
+	aggregatorClient *aggregatorclient.Clientset,
+	placeholderClient *placeholderclientset.Clientset,
+	err error,
+) {
+	// Load the Kubernetes client configuration (kubeconfig),
+	kubeConfig, err := restclient.InClusterConfig()
+	if err != nil {
+		return nil, nil, nil, fmt.Errorf("could not load in-cluster configuration: %w", err)
+	}
+
+	// explicitly use protobuf when talking to built-in kube APIs
+	protoKubeConfig := createProtoKubeConfig(kubeConfig)
+
+	// Connect to the core Kubernetes API.
+	k8sClient, err = kubernetes.NewForConfig(protoKubeConfig)
+	if err != nil {
+		return nil, nil, nil, fmt.Errorf("could not initialize Kubernetes client: %w", err)
+	}
+
+	// Connect to the Kubernetes aggregation API.
+	aggregatorClient, err = aggregatorclient.NewForConfig(protoKubeConfig)
+	if err != nil {
+		return nil, nil, nil, fmt.Errorf("could not initialize Kubernetes client: %w", err)
+	}
+
+	// Connect to the placeholder API.
+	// I think we can't use protobuf encoding here because we are using CRDs
+	// (for which protobuf encoding is not supported).
+	placeholderClient, err = placeholderclientset.NewForConfig(kubeConfig)
+	if err != nil {
+		return nil, nil, nil, fmt.Errorf("could not initialize placeholder client: %w", err)
+	}
+
+	//nolint: nakedret
+	return
+}
+
+// Create the informers that will be used by the controllers.
+func createInformers(
+	serverInstallationNamespace string,
+	k8sClient *kubernetes.Clientset,
+	placeholderClient *placeholderclientset.Clientset,
+) (
+	kubePublicNamespaceK8sInformers k8sinformers.SharedInformerFactory,
+	installationNamespaceK8sInformers k8sinformers.SharedInformerFactory,
+	installationNamespacePlaceholderInformers placeholderinformers.SharedInformerFactory,
+) {
+	kubePublicNamespaceK8sInformers = k8sinformers.NewSharedInformerFactoryWithOptions(
+		k8sClient,
+		defaultResyncInterval,
+		k8sinformers.WithNamespace(logindiscovery.ClusterInfoNamespace),
+	)
+	installationNamespaceK8sInformers = k8sinformers.NewSharedInformerFactoryWithOptions(
+		k8sClient,
+		defaultResyncInterval,
+		k8sinformers.WithNamespace(serverInstallationNamespace),
+	)
+	installationNamespacePlaceholderInformers = placeholderinformers.NewSharedInformerFactoryWithOptions(
+		placeholderClient,
+		defaultResyncInterval,
+		placeholderinformers.WithNamespace(serverInstallationNamespace),
+	)
+	return
+}
+
+// Returns a copy of the input config with the ContentConfig set to use protobuf.
+// Do not use this config to communicate with any CRD based APIs.
+func createProtoKubeConfig(kubeConfig *restclient.Config) *restclient.Config {
+	protoKubeConfig := restclient.CopyConfig(kubeConfig)
+	const protoThenJSON = runtime.ContentTypeProtobuf + "," + runtime.ContentTypeJSON
+	protoKubeConfig.AcceptContentTypes = protoThenJSON
+	protoKubeConfig.ContentType = runtime.ContentTypeProtobuf
+	return protoKubeConfig
+}

internal/provider/dynamic_tls_serving_cert_provider.go

@@ -0,0 +1,44 @@
+/*
+Copyright 2020 VMware, Inc.
+SPDX-License-Identifier: Apache-2.0
+*/
+
+package provider
+
+import (
+	"sync"
+
+	"k8s.io/apiserver/pkg/server/dynamiccertificates"
+)
+
+type DynamicTLSServingCertProvider interface {
+	dynamiccertificates.CertKeyContentProvider
+	Set(certPEM, keyPEM []byte)
+}
+
+type dynamicTLSServingCertProvider struct {
+	certPEM []byte
+	keyPEM  []byte
+	mutex   sync.RWMutex
+}
+
+func NewDynamicTLSServingCertProvider() DynamicTLSServingCertProvider {
+	return &dynamicTLSServingCertProvider{}
+}
+
+func (p *dynamicTLSServingCertProvider) Set(certPEM, keyPEM []byte) {
+	p.mutex.Lock() // acquire a write lock
+	defer p.mutex.Unlock()
+	p.certPEM = certPEM
+	p.keyPEM = keyPEM
+}
+
+func (p *dynamicTLSServingCertProvider) Name() string {
+	return "DynamicTLSServingCertProvider"
+}
+
+func (p *dynamicTLSServingCertProvider) CurrentCertKeyContent() (cert []byte, key []byte) {
+	p.mutex.RLock() // acquire a read lock
+	defer p.mutex.RUnlock()
+	return p.certPEM, p.keyPEM
+}

internal/server/server.go

@@ -8,42 +8,23 @@ package server
 
 import (
 	"context"
-	"crypto/tls"
-	"crypto/x509"
-	"crypto/x509/pkix"
-	"encoding/pem"
 	"fmt"
 	"io"
-	"time"
 
 	"github.com/spf13/cobra"
-	"k8s.io/apimachinery/pkg/runtime"
 	genericapiserver "k8s.io/apiserver/pkg/server"
-	"k8s.io/apiserver/pkg/server/dynamiccertificates"
 	genericoptions "k8s.io/apiserver/pkg/server/options"
 	"k8s.io/apiserver/plugin/pkg/authenticator/token/webhook"
-	k8sinformers "k8s.io/client-go/informers"
-	"k8s.io/client-go/kubernetes"
-	restclient "k8s.io/client-go/rest"
-	aggregationv1client "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset"
 
-	"github.com/suzerain-io/controller-go"
 	"github.com/suzerain-io/placeholder-name/internal/apiserver"
-	"github.com/suzerain-io/placeholder-name/internal/autoregistration"
 	"github.com/suzerain-io/placeholder-name/internal/certauthority"
-	"github.com/suzerain-io/placeholder-name/internal/controller/logindiscovery"
+	"github.com/suzerain-io/placeholder-name/internal/controllermanager"
 	"github.com/suzerain-io/placeholder-name/internal/downward"
+	"github.com/suzerain-io/placeholder-name/internal/provider"
 	placeholderv1alpha1 "github.com/suzerain-io/placeholder-name/kubernetes/1.19/api/apis/placeholder/v1alpha1"
-	placeholderclientset "github.com/suzerain-io/placeholder-name/kubernetes/1.19/client-go/clientset/versioned"
-	placeholderinformers "github.com/suzerain-io/placeholder-name/kubernetes/1.19/client-go/informers/externalversions"
 	"github.com/suzerain-io/placeholder-name/pkg/config"
 )
 
-const (
-	singletonWorker       = 1
-	defaultResyncInterval = 3 * time.Minute
-)
-
 // App is an object that represents the placeholder-name-server application.
 type App struct {
 	cmd *cobra.Command
@@ -53,8 +34,6 @@ type App struct {
 	downwardAPIPath            string
 	clusterSigningCertFilePath string
 	clusterSigningKeyFilePath  string
-
-	recommendedOptions *genericoptions.RecommendedOptions
 }
 
 // This is ignored for now because we turn off etcd storage below, but this is
@@ -63,61 +42,39 @@ const defaultEtcdPathPrefix = "/registry/" + placeholderv1alpha1.GroupName
 
 // New constructs a new App with command line args, stdout and stderr.
 func New(ctx context.Context, args []string, stdout, stderr io.Writer) *App {
-	a := &App{
-		recommendedOptions: genericoptions.NewRecommendedOptions(
-			defaultEtcdPathPrefix,
-			apiserver.Codecs.LegacyCodec(placeholderv1alpha1.SchemeGroupVersion),
-			// TODO we should check to see if all the other default settings are acceptable for us
-		),
-	}
-	a.recommendedOptions.Etcd = nil // turn off etcd storage because we don't need it yet
+	app := &App{}
+	app.addServerCommand(ctx, args, stdout, stderr)
+	return app
+}
 
+// Run the server.
+func (a *App) Run() error {
+	return a.cmd.Execute()
+}
+
+// Create the server command and save it into the App.
+func (a *App) addServerCommand(ctx context.Context, args []string, stdout, stderr io.Writer) {
 	cmd := &cobra.Command{
 		Use: `placeholder-name-server`,
-		Long: `placeholder-name-server provides a generic API for mapping an external
-credential from somewhere to an internal credential to be used for
-authenticating to the Kubernetes API.`,
-		RunE: func(cmd *cobra.Command, args []string) error {
-			// Load the Kubernetes client configuration (kubeconfig),
-			kubeConfig, err := restclient.InClusterConfig()
-			if err != nil {
-				return fmt.Errorf("could not load in-cluster configuration: %w", err)
-			}
-
-			// explicitly use protobuf when talking to built-in kube APIs
-			protoKubeConfig := createProtoKubeConfig(kubeConfig)
-
-			// Connect to the core Kubernetes API.
-			k8sClient, err := kubernetes.NewForConfig(protoKubeConfig)
-			if err != nil {
-				return fmt.Errorf("could not initialize Kubernetes client: %w", err)
-			}
-
-			// Connect to the Kubernetes aggregation API.
-			aggregationClient, err := aggregationv1client.NewForConfig(protoKubeConfig)
-			if err != nil {
-				return fmt.Errorf("could not initialize Kubernetes client: %w", err)
-			}
-
-			// Connect to the placeholder API.
-			// I think we can't use protobuf encoding here because we are using CRDs
-			// (for which protobuf encoding is not supported).
-			placeholderClient, err := placeholderclientset.NewForConfig(kubeConfig)
-			if err != nil {
-				return fmt.Errorf("could not initialize placeholder client: %w", err)
-			}
-
-			return a.run(ctx, k8sClient, aggregationClient, placeholderClient)
-		},
+		Long: "placeholder-name-server provides a generic API for mapping an external\n" +
+			"credential from somewhere to an internal credential to be used for\n" +
+			"authenticating to the Kubernetes API.",
+		RunE: func(cmd *cobra.Command, args []string) error { return a.runServer(ctx) },
 		Args: cobra.NoArgs,
 	}
 
 	cmd.SetArgs(args)
 	cmd.SetOut(stdout)
 	cmd.SetErr(stderr)
+	addCommandlineFlagsToCommand(cmd, a)
 
+	a.cmd = cmd
+}
+
+// Define the app's commandline flags.
+func addCommandlineFlagsToCommand(cmd *cobra.Command, app *App) {
 	cmd.Flags().StringVarP(
-		&a.configPath,
+		&app.configPath,
 		"config",
 		"c",
 		"placeholder-name.yaml",
@@ -125,123 +82,119 @@ authenticating to the Kubernetes API.`,
 	)
 
 	cmd.Flags().StringVar(
-		&a.downwardAPIPath,
+		&app.downwardAPIPath,
 		"downward-api-path",
 		"/etc/podinfo",
 		"path to Downward API volume mount",
 	)
 
 	cmd.Flags().StringVar(
-		&a.clusterSigningCertFilePath,
+		&app.clusterSigningCertFilePath,
 		"cluster-signing-cert-file",
 		"",
 		"path to cluster signing certificate",
 	)
 
 	cmd.Flags().StringVar(
-		&a.clusterSigningKeyFilePath,
+		&app.clusterSigningKeyFilePath,
 		"cluster-signing-key-file",
 		"",
 		"path to cluster signing private key",
 	)
-
-	a.cmd = cmd
-
-	return a
 }
 
-func (a *App) Run() error {
-	return a.cmd.Execute()
-}
-
-func (a *App) run(
-	ctx context.Context,
-	k8sClient kubernetes.Interface,
-	aggregationClient aggregationv1client.Interface,
-	placeholderClient placeholderclientset.Interface,
-) error {
+// Boot the aggregated API server, which will in turn boot the controllers.
+func (a *App) runServer(ctx context.Context) error {
+	// Read the server config file.
 	cfg, err := config.FromPath(a.configPath)
 	if err != nil {
 		return fmt.Errorf("could not load config: %w", err)
 	}
 
 	// Load the Kubernetes cluster signing CA.
-	clientCA, err := certauthority.Load(a.clusterSigningCertFilePath, a.clusterSigningKeyFilePath)
+	k8sClusterCA, err := certauthority.Load(a.clusterSigningCertFilePath, a.clusterSigningKeyFilePath)
 	if err != nil {
 		return fmt.Errorf("could not load cluster signing CA: %w", err)
 	}
 
+	// Create a WebhookTokenAuthenticator.
 	webhookTokenAuthenticator, err := config.NewWebhook(cfg.WebhookConfig)
 	if err != nil {
 		return fmt.Errorf("could not create webhook client: %w", err)
 	}
 
-	podinfo, err := downward.Load(a.downwardAPIPath)
+	// Discover in which namespace we are installed.
+	podInfo, err := downward.Load(a.downwardAPIPath)
 	if err != nil {
 		return fmt.Errorf("could not read pod metadata: %w", err)
 	}
-	serverInstallationNamespace := podinfo.Namespace
+	serverInstallationNamespace := podInfo.Namespace
 
-	// TODO use the postStart hook to generate certs?
+	// This cert provider will provide certs to the API server and will
+	// be mutated by a controller to keep the certs up to date with what
+	// is stored in a k8s Secret. Therefore it also effectively acting as
+	// an in-memory cache of what is stored in the k8s Secret, helping to
+	// keep incoming requests fast.
+	dynamicCertProvider := provider.NewDynamicTLSServingCertProvider()
 
-	aggregatedAPIServerCA, err := certauthority.New(pkix.Name{CommonName: "Placeholder CA"})
-	if err != nil {
-		return fmt.Errorf("could not initialize CA: %w", err)
-	}
-
-	const serviceName = "placeholder-name-api"
-
-	cert, err := aggregatedAPIServerCA.Issue(
-		pkix.Name{CommonName: serviceName + "." + serverInstallationNamespace + ".svc"},
-		[]string{},
-		24*365*time.Hour,
-	)
-	if err != nil {
-		return fmt.Errorf("could not issue serving certificate: %w", err)
-	}
-
-	if err := autoregistration.UpdateAPIService(ctx, aggregationClient, aggregatedAPIServerCA.Bundle()); err != nil {
-		return fmt.Errorf("could not register API service: %w", err)
-	}
-
-	cmrf := wireControllerManagerRunFunc(
+	// Prepare to start the controllers, but defer actually starting them until the
+	// post start hook of the aggregated API server.
+	startControllersFunc, err := controllermanager.PrepareControllers(
 		serverInstallationNamespace,
 		cfg.DiscoveryConfig.URL,
-		k8sClient,
-		placeholderClient,
+		dynamicCertProvider,
 	)
-	apiServerConfig, err := a.configServer(
-		cert,
+	if err != nil {
+		return fmt.Errorf("could not prepare controllers: %w", err)
+	}
+
+	// Get the aggregated API server config.
+	aggregatedAPIServerConfig, err := getAggregatedAPIServerConfig(
+		dynamicCertProvider,
 		webhookTokenAuthenticator,
-		clientCA,
-		cmrf,
+		k8sClusterCA,
+		startControllersFunc,
 	)
 	if err != nil {
-		return err
+		return fmt.Errorf("could not configure aggregated API server: %w", err)
 	}
 
-	server, err := apiServerConfig.Complete().New()
+	// Complete the aggregated API server config and make a server instance.
+	server, err := aggregatedAPIServerConfig.Complete().New()
 	if err != nil {
-		return fmt.Errorf("could not issue serving certificate: %w", err)
+		return fmt.Errorf("could not create aggregated API server: %w", err)
 	}
 
+	// Run the server. Its post-start hook will start the controllers.
 	return server.GenericAPIServer.PrepareRun().Run(ctx.Done())
 }
 
-func (a *App) configServer(
-	cert *tls.Certificate,
+// Create a configuration for the aggregated API server.
+func getAggregatedAPIServerConfig(
+	dynamicCertProvider provider.DynamicTLSServingCertProvider,
 	webhookTokenAuthenticator *webhook.WebhookTokenAuthenticator,
 	ca *certauthority.CA,
 	startControllersPostStartHook func(context.Context),
 ) (*apiserver.Config, error) {
-	provider, err := createStaticCertKeyProvider(cert)
-	if err != nil {
-		return nil, fmt.Errorf("could not create static cert key provider: %w", err)
-	}
-	a.recommendedOptions.SecureServing.ServerCert.GeneratedCert = provider
+	recommendedOptions := genericoptions.NewRecommendedOptions(
+		defaultEtcdPathPrefix,
+		apiserver.Codecs.LegacyCodec(placeholderv1alpha1.SchemeGroupVersion),
+		// TODO we should check to see if all the other default settings are acceptable for us
+	)
+	recommendedOptions.Etcd = nil // turn off etcd storage because we don't need it yet
+	recommendedOptions.SecureServing.ServerCert.GeneratedCert = dynamicCertProvider
 
 	serverConfig := genericapiserver.NewRecommendedConfig(apiserver.Codecs)
-	if err := a.recommendedOptions.ApplyTo(serverConfig); err != nil {
+	// Note that among other things, this ApplyTo() function copies
+	// `recommendedOptions.SecureServing.ServerCert.GeneratedCert` into
+	// `serverConfig.SecureServing.Cert` thus making `dynamicCertProvider`
+	// the cert provider for the running server. The provider will be called
+	// by the API machinery periodically. When the provider returns nil certs,
+	// the API server will return "the server is currently unable to
+	// handle the request" error responses for all incoming requests.
+	// If the provider later starts returning certs, then the API server
+	// will use them to handle the incoming requests successfully.
+	if err := recommendedOptions.ApplyTo(serverConfig); err != nil {
 		return nil, err
 	}
 
@@ -255,75 +208,3 @@ func (a *App) configServer(
 	}
 	return apiServerConfig, nil
 }
-
-// createProtoKubeConfig returns a copy of the input config with the ContentConfig set to use protobuf.
-// do not use this config to communicate with any CRD based APIs.
-func createProtoKubeConfig(kubeConfig *restclient.Config) *restclient.Config {
-	protoKubeConfig := restclient.CopyConfig(kubeConfig)
-	const protoThenJSON = runtime.ContentTypeProtobuf + "," + runtime.ContentTypeJSON
-	protoKubeConfig.AcceptContentTypes = protoThenJSON
-	protoKubeConfig.ContentType = runtime.ContentTypeProtobuf
-	return protoKubeConfig
-}
-
-func createStaticCertKeyProvider(cert *tls.Certificate) (dynamiccertificates.CertKeyContentProvider, error) {
-	privateKeyDER, err := x509.MarshalPKCS8PrivateKey(cert.PrivateKey)
-	if err != nil {
-		return nil, fmt.Errorf("error marshalling private key: %w", err)
-	}
-	privateKeyPEM := pem.EncodeToMemory(&pem.Block{
-		Type:    "PRIVATE KEY",
-		Headers: nil,
-		Bytes:   privateKeyDER,
-	})
-
-	certChainPEM := make([]byte, 0)
-	for _, certFromChain := range cert.Certificate {
-		certPEMBytes := pem.EncodeToMemory(&pem.Block{
-			Type:    "CERTIFICATE",
-			Headers: nil,
-			Bytes:   certFromChain,
-		})
-		certChainPEM = append(certChainPEM, certPEMBytes...)
-	}
-
-	return dynamiccertificates.NewStaticCertKeyContent("some-name???", certChainPEM, privateKeyPEM)
-}
-
-func wireControllerManagerRunFunc(
-	serverInstallationNamespace string,
-	discoveryURLOverride *string,
-	k8s kubernetes.Interface,
-	placeholder placeholderclientset.Interface,
-) func(ctx context.Context) {
-	k8sInformers := k8sinformers.NewSharedInformerFactoryWithOptions(
-		k8s,
-		defaultResyncInterval,
-		k8sinformers.WithNamespace(
-			logindiscovery.ClusterInfoNamespace,
-		),
-	)
-	placeholderInformers := placeholderinformers.NewSharedInformerFactoryWithOptions(
-		placeholder,
-		defaultResyncInterval,
-		placeholderinformers.WithNamespace(serverInstallationNamespace),
-	)
-	cm := controller.
-		NewManager().
-		WithController(
-			logindiscovery.NewPublisherController(
-				serverInstallationNamespace,
-				discoveryURLOverride,
-				placeholder,
-				k8sInformers.Core().V1().ConfigMaps(),
-				placeholderInformers.Crds().V1alpha1().LoginDiscoveryConfigs(),
-				controller.WithInformer,
-			),
-			singletonWorker,
-		)
-	return func(ctx context.Context) {
-		k8sInformers.Start(ctx.Done())
-		placeholderInformers.Start(ctx.Done())
-		go cm.Start(ctx)
-	}
-}

internal/testutil/observable_with_informer_option.go

@@ -0,0 +1,31 @@
+/*
+Copyright 2020 VMware, Inc.
+SPDX-License-Identifier: Apache-2.0
+*/
+
+package testutil
+
+import "github.com/suzerain-io/controller-go"
+
+type ObservableWithInformerOption struct {
+	informerToFilterMap map[controller.InformerGetter]controller.Filter
+}
+
+func NewObservableWithInformerOption() *ObservableWithInformerOption {
+	return &ObservableWithInformerOption{
+		informerToFilterMap: make(map[controller.InformerGetter]controller.Filter),
+	}
+}
+
+func (i *ObservableWithInformerOption) WithInformer(
+	getter controller.InformerGetter,
+	filter controller.Filter,
+	opt controller.InformerOption,
+) controller.Option {
+	i.informerToFilterMap[getter] = filter
+	return controller.WithInformer(getter, filter, opt)
+}
+
+func (i *ObservableWithInformerOption) GetFilterForInformer(getter controller.InformerGetter) controller.Filter {
+	return i.informerToFilterMap[getter]
+}

internal/testutil/observable_with_initial_event_option.go

@@ -0,0 +1,25 @@
+/*
+Copyright 2020 VMware, Inc.
+SPDX-License-Identifier: Apache-2.0
+*/
+
+package testutil
+
+import "github.com/suzerain-io/controller-go"
+
+type ObservableWithInitialEventOption struct {
+	key controller.Key
+}
+
+func NewObservableWithInitialEventOption() *ObservableWithInitialEventOption {
+	return &ObservableWithInitialEventOption{}
+}
+
+func (i *ObservableWithInitialEventOption) WithInitialEvent(key controller.Key) controller.Option {
+	i.key = key
+	return controller.WithInitialEvent(key)
+}
+
+func (i *ObservableWithInitialEventOption) GetInitialEventKey() controller.Key {
+	return i.key
+}

test/go.mod

@@ -11,6 +11,7 @@ require (
 	k8s.io/api v0.19.0-rc.0
 	k8s.io/apimachinery v0.19.0-rc.0
 	k8s.io/client-go v0.19.0-rc.0
+	k8s.io/kube-aggregator v0.18.6
 )
 
 replace (

test/go.sum

@@ -11,6 +11,7 @@ cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7
 cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
 cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
+github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
 github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI=
 github.com/Azure/go-autorest/autorest v0.9.6/go.mod h1:/FALq9T/kS7b5J5qsQ+RSTUdAmGFqi0vUdVNNx8q630=
 github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0=
@@ -30,29 +31,56 @@ github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbt
 github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
 github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
 github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
+github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
+github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
+github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
+github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
+github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
+github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
+github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc=
+github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
+github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
+github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
+github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
+github.com/docker/docker v0.7.3-0.20190327010347-be7ac8be2ae0/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
 github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
+github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
 github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/evanphx/json-patch v0.0.0-20190815234213-e83c0a1c26c8/go.mod h1:pmLOTb3x90VhIKxsA9yeQG5yfOkkKnkk1h+Ql8NDYDw=
+github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
+github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
+github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
 github.com/go-logr/logr v0.2.0 h1:QvGt2nLcHH0WK9orKa+ppBPAxREcH364nPUedEpK0TY=
 github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
@@ -67,14 +95,19 @@ github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8
 github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I=
 github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
 github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
+github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
 github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls=
 github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
+github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -104,28 +137,46 @@ github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OI
 github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
+github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
+github.com/googleapis/gnostic v0.1.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
 github.com/googleapis/gnostic v0.4.1 h1:DLJCy1n/vrD4HPjOvYcT8aYQXpPIzoRZONaYwyycI+I=
 github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg=
+github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8=
+github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
+github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
+github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
+github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
+github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q=
 github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
+github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns=
 github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
 github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
+github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
 github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs=
 github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
@@ -135,10 +186,17 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
+github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
+github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
+github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
+github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -146,9 +204,12 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN
 github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
+github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
 github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
@@ -157,29 +218,68 @@ github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGV
 github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
 github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA=
+github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA=
+github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
+github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
+github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
+github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
+github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
+github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
 github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc=
 github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
+github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
+github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
+github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
+github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
 github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48=
 github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
+github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
+github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
+github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
+github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
+go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
+go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
+go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
+go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
+golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
@@ -209,9 +309,12 @@ golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCc
 golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
 golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
 golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
@@ -221,7 +324,9 @@ golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k=
 golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
@@ -238,34 +343,46 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4 h1:5/PjkGUjvEU5Gl6BxmvKRPpqo2uNMv4rcHBMwzk/st8=
 golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e h1:EHBhcS0mlXEAVwNyO2dLfjToGsyY4j24pTs2ScHnX7s=
 golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -283,6 +400,7 @@ golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgw
 golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
@@ -315,6 +433,7 @@ google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZi
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
 google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
 google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
 google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
@@ -327,17 +446,23 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2
 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.24.0 h1:UhZDfRO8JRQru4/+LlLE0BRKGF8L+PICnvYZmx/fEGA=
 google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
+gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
+gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
+gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
@@ -347,26 +472,44 @@ gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
+k8s.io/api v0.18.6/go.mod h1:eeyxr+cwCjMdLAmr2W3RyDI0VvTawSg/3RFFBEnmZGI=
 k8s.io/api v0.19.0-rc.0 h1:K+xi+F3RNAxpFyS1f7uHekMNprjFX7WVZDx2lJE+A3A=
 k8s.io/api v0.19.0-rc.0/go.mod h1:WBGMHEmngOdQBAvJiYUgP5mGDdCWXM52yDm1gtos8C0=
+k8s.io/apimachinery v0.18.6/go.mod h1:OaXp26zu/5J7p0f92ASynJa1pZo06YlV9fG7BoWbCko=
 k8s.io/apimachinery v0.19.0-rc.0 h1:IBmRy0elJCGgxtCT0bHT93N+rhx+vF2DD1XXJ3ntLa8=
 k8s.io/apimachinery v0.19.0-rc.0/go.mod h1:EjWiYOPi+BZennZ5pGa3JLkQ+znhEOodGy/+umjiLDU=
+k8s.io/apiserver v0.18.6/go.mod h1:Zt2XvTHuaZjBz6EFYzpp+X4hTmgWGy8AthNVnTdm3Wg=
+k8s.io/client-go v0.18.6/go.mod h1:/fwtGLjYMS1MaM5oi+eXhKwG+1UHidUEXRh6cNsdO0Q=
 k8s.io/client-go v0.19.0-rc.0 h1:6WW8MElhoLeYcLiN4ky1159XG5E39KYdmLCrV/6lNiE=
 k8s.io/client-go v0.19.0-rc.0/go.mod h1:3kWGD05F7c58atlk7ep9ob1hg2Yu9NSz8gJxCNNTHhc=
+k8s.io/code-generator v0.18.6/go.mod h1:TgNEVx9hCyPGpdtCWA34olQYLkh3ok9ar7XfSsr8b6c=
+k8s.io/component-base v0.18.6/go.mod h1:knSVsibPR5K6EW2XOjEHik6sdU5nCvKMrzMt2D4In14=
+k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
+k8s.io/gengo v0.0.0-20200114144118-36b2048a9120/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
 k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
+k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
+k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
+k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8=
+k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=
 k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
 k8s.io/klog/v2 v2.2.0 h1:XRvcwJozkgZ1UQJmfMGpvRthQHOvihEhYtDfAaxMz/A=
 k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
+k8s.io/kube-aggregator v0.18.6 h1:xGP3oe0tAWEYnGWTnDPjXiIItekrnwDA2O7w0WqvGoo=
+k8s.io/kube-aggregator v0.18.6/go.mod h1:MKm8inLHdeiXQJCl6UdmgMosRrqJgyxO2obTXOkey/s=
+k8s.io/kube-openapi v0.0.0-20200410145947-61e04a5be9a6/go.mod h1:GRQhZsXIAJ1xR0C9bd8UpWHZ5plfAS9fzPjJuQ6JL3E=
 k8s.io/kube-openapi v0.0.0-20200427153329-656914f816f9/go.mod h1:bfCVj+qXcEaE5SCvzBaqpOySr6tuCcpPKqF6HD8nyCw=
 k8s.io/kube-openapi v0.0.0-20200615155156-dffdd1682719/go.mod h1:bfCVj+qXcEaE5SCvzBaqpOySr6tuCcpPKqF6HD8nyCw=
+k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew=
 k8s.io/utils v0.0.0-20200619165400-6e3d28b6ed19 h1:7Nu2dTj82c6IaWvL7hImJzcXoTPz1MsSCH7r+0m6rfo=
 k8s.io/utils v0.0.0-20200619165400-6e3d28b6ed19/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
+sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.7/go.mod h1:PHgbrJT7lCHcxMU+mDHEm+nx46H4zuuHZkDP6icnhu0=
 sigs.k8s.io/structured-merge-diff/v3 v3.0.0-20200116222232-67a7b8c61874/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw=
 sigs.k8s.io/structured-merge-diff/v3 v3.0.0 h1:dOmIZBMfhcHS09XZkMyUgkq5trg3/jRyJYFZUiaOp8E=
 sigs.k8s.io/structured-merge-diff/v3 v3.0.0/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw=

test/integration/api_serving_certs_test.go

@@ -0,0 +1,97 @@
+/*
+Copyright 2020 VMware, Inc.
+SPDX-License-Identifier: Apache-2.0
+*/
+
+package integration
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/suzerain-io/placeholder-name/kubernetes/1.19/api/apis/placeholder/v1alpha1"
+	"github.com/suzerain-io/placeholder-name/test/library"
+)
+
+func TestAPIServingCertificateAutoCreationAndRotation(t *testing.T) {
+	library.SkipUnlessIntegration(t)
+	namespaceName := library.Getenv(t, "PLACEHOLDER_NAME_NAMESPACE")
+
+	kubeClient := library.NewClientset(t)
+	aggregatedClient := library.NewAggregatedClientset(t)
+	placeholderClient := library.NewPlaceholderNameClientset(t)
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+	defer cancel()
+
+	const apiServiceName = "v1alpha1.placeholder.suzerain-io.github.io"
+
+	// Get the initial auto-generated version of the Secret.
+	secret, err := kubeClient.CoreV1().Secrets(namespaceName).Get(ctx, "api-serving-cert", metav1.GetOptions{})
+	require.NoError(t, err)
+	initialCACert := secret.Data["caCertificate"]
+	initialPrivateKey := secret.Data["tlsPrivateKey"]
+	initialCertChain := secret.Data["tlsCertificateChain"]
+	require.NotEmpty(t, initialCACert)
+	require.NotEmpty(t, initialPrivateKey)
+	require.NotEmpty(t, initialCertChain)
+
+	// Check that the APIService has the same CA.
+	apiService, err := aggregatedClient.ApiregistrationV1().APIServices().Get(ctx, apiServiceName, metav1.GetOptions{})
+	require.NoError(t, err)
+	require.Equal(t, initialCACert, apiService.Spec.CABundle)
+
+	// Delete the Secret, simulating an end user doing `kubectl delete` to manually ask for an immediate rotation.
+	err = kubeClient.CoreV1().Secrets(namespaceName).Delete(ctx, "api-serving-cert", metav1.DeleteOptions{})
+	require.NoError(t, err)
+
+	// Expect that the Secret comes back right away with newly minted certs.
+	var secretIsRegenerated = func() bool {
+		secret, err = kubeClient.CoreV1().Secrets(namespaceName).Get(ctx, "api-serving-cert", metav1.GetOptions{})
+		return err == nil
+	}
+	assert.Eventually(t, secretIsRegenerated, 10*time.Second, 250*time.Millisecond)
+	require.NoError(t, err) // prints out the error in case of failure
+	regeneratedCACert := secret.Data["caCertificate"]
+	regeneratedPrivateKey := secret.Data["tlsPrivateKey"]
+	regeneratedCertChain := secret.Data["tlsCertificateChain"]
+	require.NotEmpty(t, regeneratedCACert)
+	require.NotEmpty(t, regeneratedPrivateKey)
+	require.NotEmpty(t, regeneratedCertChain)
+	require.NotEqual(t, initialCACert, regeneratedCACert)
+	require.NotEqual(t, initialPrivateKey, regeneratedPrivateKey)
+	require.NotEqual(t, initialCertChain, regeneratedCertChain)
+
+	// Expect that the APIService was also updated with the new CA.
+	var aggregatedAPIUpdated = func() bool {
+		apiService, err = aggregatedClient.ApiregistrationV1().APIServices().Get(ctx, apiServiceName, metav1.GetOptions{})
+		return err == nil
+	}
+	assert.Eventually(t, aggregatedAPIUpdated, 10*time.Second, 250*time.Millisecond)
+	require.NoError(t, err) // prints out the error in case of failure
+	require.Equal(t, regeneratedCACert, apiService.Spec.CABundle)
+
+	// Check that we can still make requests to the aggregated API through the kube API server,
+	// because the kube API server uses these certs when proxying requests to the aggregated API server,
+	// so this is effectively checking that the aggregated API server is using these new certs.
+	var aggregatedAPIWorking = func() bool {
+		_, err = placeholderClient.PlaceholderV1alpha1().LoginRequests().Create(ctx, &v1alpha1.LoginRequest{
+			TypeMeta:   metav1.TypeMeta{},
+			ObjectMeta: metav1.ObjectMeta{},
+			Spec: v1alpha1.LoginRequestSpec{
+				Type:  v1alpha1.TokenLoginCredentialType,
+				Token: &v1alpha1.LoginRequestTokenCredential{Value: "not a good token"},
+			},
+		}, metav1.CreateOptions{})
+		// Should have got a success response with an error message inside it complaining about the token value.
+		return err == nil
+	}
+	// Unfortunately, although our code changes all the certs immediately, it seems to take ~1 minute for
+	// the API machinery to notice that we updated our serving cert, causing 1 minute of downtime for our endpoint.
+	assert.Eventually(t, aggregatedAPIWorking, 2*time.Minute, 250*time.Millisecond)
+	require.NoError(t, err) // prints out the error in case of failure
+}

test/library/client.go

@@ -13,6 +13,7 @@ import (
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
 	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
+	aggregatorclient "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset"
 
 	placeholdernameclientset "github.com/suzerain-io/placeholder-name/kubernetes/1.19/client-go/clientset/versioned"
 )
@@ -63,3 +64,9 @@ func NewPlaceholderNameClientset(t *testing.T) placeholdernameclientset.Interfac
 
 	return placeholdernameclientset.NewForConfigOrDie(NewClientConfig(t))
 }
+
+func NewAggregatedClientset(t *testing.T) aggregatorclient.Interface {
+	t.Helper()
+
+	return aggregatorclient.NewForConfigOrDie(NewClientConfig(t))
+}