Auto-rotate serving certificate

The rotation is forced by a new controller that deletes the serving cert
secret, as other controllers will see this deletion and ensure that a new
serving cert is created.

Note that the integration tests now have an addition worst case runtime of
60 seconds. This is because of the way that the aggregated API server code
reloads certificates. We will fix this in a future story. Then, the
integration tests should hopefully get much faster.

Signed-off-by: Andrew Keesler <akeesler@vmware.com>

deploy/rbac.yaml

@@ -43,7 +43,7 @@ rules:
     verbs: [create, get, list, patch, update, watch]
   - apiGroups: [""]
     resources: [secrets]
-    verbs: [create, get, list, patch, update, watch]
+    verbs: [create, get, list, patch, update, watch, delete]
   - apiGroups: [crds.placeholder.suzerain-io.github.io]
     resources: [logindiscoveryconfigs]
     verbs: [create, get, list, update, watch]

internal/controller/apicerts/certs_expirer.go

@@ -0,0 +1,128 @@
+/*
+Copyright 2020 VMware, Inc.
+SPDX-License-Identifier: Apache-2.0
+*/
+
+package apicerts
+
+import (
+	"crypto/x509"
+	"encoding/pem"
+	"fmt"
+	"time"
+
+	corev1 "k8s.io/api/core/v1"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	corev1informers "k8s.io/client-go/informers/core/v1"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/klog/v2"
+
+	"github.com/suzerain-io/controller-go"
+	"github.com/suzerain-io/placeholder-name/internal/constable"
+	placeholdernamecontroller "github.com/suzerain-io/placeholder-name/internal/controller"
+)
+
+type certsExpirerController struct {
+	namespace      string
+	k8sClient      kubernetes.Interface
+	secretInformer corev1informers.SecretInformer
+
+	// ageThreshold is a percentage (i.e., a real number between 0 and 1,
+	// inclusive) indicating the point in a certificate's lifetime where this
+	// controller will start to try to rotate it.
+	//
+	// Said another way, once ageThreshold % of a certificate's lifetime has
+	// passed, this controller will try to delete it to force a new certificate
+	// to be created.
+	ageThreshold float32
+}
+
+// NewCertsExpirerController returns a controller.Controller that will delete a
+// CA once it gets within some threshold of its expiration time.
+func NewCertsExpirerController(
+	namespace string,
+	k8sClient kubernetes.Interface,
+	secretInformer corev1informers.SecretInformer,
+	withInformer placeholdernamecontroller.WithInformerOptionFunc,
+	ageThreshold float32,
+) controller.Controller {
+	return controller.New(
+		controller.Config{
+			Name: "certs-expirer-controller",
+			Syncer: &certsExpirerController{
+				namespace:      namespace,
+				k8sClient:      k8sClient,
+				secretInformer: secretInformer,
+				ageThreshold:   ageThreshold,
+			},
+		},
+		withInformer(
+			secretInformer,
+			placeholdernamecontroller.NameAndNamespaceExactMatchFilterFactory(certsSecretName, namespace),
+			controller.InformerOption{},
+		),
+	)
+}
+
+// Sync implements controller.Syncer.Sync.
+func (c *certsExpirerController) Sync(ctx controller.Context) error {
+	secret, err := c.secretInformer.Lister().Secrets(c.namespace).Get(certsSecretName)
+	notFound := k8serrors.IsNotFound(err)
+	if err != nil && !notFound {
+		return fmt.Errorf("failed to get %s/%s secret: %w", c.namespace, certsSecretName, err)
+	}
+	if notFound {
+		klog.Info("certsExpirerController Sync() found that the secret does not exist yet or was deleted")
+		return nil
+	}
+
+	notBefore, notAfter, err := getCABounds(secret)
+	if err != nil {
+		// If we can't get the CA, then really all we can do is log something, since
+		// if we returned an error then the controller lib would just call us again
+		// and again, which would probably yield the same results.
+		klog.Warningf("certsExpirerController Sync() found that the secret is malformed: %s", err.Error())
+		return nil
+	}
+
+	caLifetime := notAfter.Sub(notBefore)
+	caAge := time.Since(notBefore)
+	thresholdDelta := (float32(caAge) / float32(caLifetime)) - c.ageThreshold
+	klog.Infof("certsExpirerController Sync() found a CA age threshold delta of %.2f", thresholdDelta)
+	if thresholdDelta > 0 {
+		err := c.k8sClient.
+			CoreV1().
+			Secrets(c.namespace).
+			Delete(ctx.Context, certsSecretName, metav1.DeleteOptions{})
+		if err != nil {
+			// Do return an error here so that the controller library will reschedule
+			// us to try deleting this cert again.
+			return err
+		}
+	}
+
+	return nil
+}
+
+// getCABounds returns the NotBefore and NotAfter fields of the CA certificate
+// in the provided secret, or an error. Not that it expects the provided secret
+// to contain the well-known data keys from this package (see certs_manager.go).
+func getCABounds(secret *corev1.Secret) (time.Time, time.Time, error) {
+	caPEM := secret.Data[caCertificateSecretKey]
+	if caPEM == nil {
+		return time.Time{}, time.Time{}, constable.Error("failed to find CA")
+	}
+
+	caBlock, _ := pem.Decode(caPEM)
+	if caBlock == nil {
+		return time.Time{}, time.Time{}, constable.Error("failed to decode CA PEM")
+	}
+
+	caCrt, err := x509.ParseCertificate(caBlock.Bytes)
+	if err != nil {
+		return time.Time{}, time.Time{}, fmt.Errorf("failed to parse CA: %w", err)
+	}
+
+	return caCrt.NotBefore, caCrt.NotAfter, nil
+}

internal/controller/apicerts/certs_expirer_test.go

@@ -0,0 +1,269 @@
+/*
+Copyright 2020 VMware, Inc.
+SPDX-License-Identifier: Apache-2.0
+*/
+
+package apicerts
+
+import (
+	"context"
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/x509"
+	"errors"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	kubeinformers "k8s.io/client-go/informers"
+	kubernetesfake "k8s.io/client-go/kubernetes/fake"
+	kubetesting "k8s.io/client-go/testing"
+
+	"github.com/suzerain-io/controller-go"
+	"github.com/suzerain-io/placeholder-name/internal/testutil"
+)
+
+func TestExpirerControllerFilters(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name      string
+		namespace string
+		secret    corev1.Secret
+		want      bool
+	}{
+		{
+			name:      "good name, good namespace",
+			namespace: "good-namespace",
+			secret: corev1.Secret{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "api-serving-cert",
+					Namespace: "good-namespace",
+				},
+			},
+			want: true,
+		},
+		{
+			name:      "bad name, good namespace",
+			namespace: "good-namespacee",
+			secret: corev1.Secret{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "bad-name",
+					Namespace: "good-namespace",
+				},
+			},
+			want: false,
+		},
+		{
+			name:      "good name, bad namespace",
+			namespace: "good-namespacee",
+			secret: corev1.Secret{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "api-serving-cert",
+					Namespace: "bad-namespace",
+				},
+			},
+			want: false,
+		},
+		{
+			name:      "bad name, bad namespace",
+			namespace: "good-namespacee",
+			secret: corev1.Secret{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "bad-name",
+					Namespace: "bad-namespace",
+				},
+			},
+			want: false,
+		},
+	}
+	for _, test := range tests {
+		test := test
+		t.Run(test.name+"-"+test.namespace, func(t *testing.T) {
+			t.Parallel()
+
+			secretsInformer := kubeinformers.NewSharedInformerFactory(
+				kubernetesfake.NewSimpleClientset(),
+				0,
+			).Core().V1().Secrets()
+			withInformer := testutil.NewObservableWithInformerOption()
+			_ = NewCertsExpirerController(
+				test.namespace,
+				nil, // k8sClient, not needed
+				secretsInformer,
+				withInformer.WithInformer,
+				0, // ageThreshold, not needed
+			)
+
+			unrelated := corev1.Secret{}
+			filter := withInformer.GetFilterForInformer(secretsInformer)
+			require.Equal(t, test.want, filter.Add(&test.secret))
+			require.Equal(t, test.want, filter.Update(&unrelated, &test.secret))
+			require.Equal(t, test.want, filter.Update(&test.secret, &unrelated))
+			require.Equal(t, test.want, filter.Delete(&test.secret))
+		})
+	}
+}
+
+func TestExpirerControllerSync(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name                string
+		ageThreshold        float32
+		fillSecretData      func(*testing.T, map[string][]byte)
+		configKubeAPIClient func(*kubernetesfake.Clientset)
+		wantDelete          bool
+		wantError           string
+	}{
+		{
+			name:       "secret does not exist",
+			wantDelete: false,
+		},
+		{
+			name:           "secret missing key",
+			fillSecretData: func(t *testing.T, m map[string][]byte) {},
+			wantDelete:     false,
+		},
+		{
+			name:         "lifetime below threshold",
+			ageThreshold: 0.7,
+			fillSecretData: func(t *testing.T, m map[string][]byte) {
+				caPEM, err := testutil.CreateCertificate(
+					time.Now().Add(-5*time.Hour),
+					time.Now().Add(5*time.Hour),
+				)
+				require.NoError(t, err)
+
+				// See cert_manager.go for this constant.
+				m["caCertificate"] = caPEM
+			},
+			wantDelete: false,
+		},
+		{
+			name:         "lifetime above threshold",
+			ageThreshold: 0.3,
+			fillSecretData: func(t *testing.T, m map[string][]byte) {
+				caPEM, err := testutil.CreateCertificate(
+					time.Now().Add(-5*time.Hour),
+					time.Now().Add(5*time.Hour),
+				)
+				require.NoError(t, err)
+
+				// See cert_manager.go for this constant.
+				m["caCertificate"] = caPEM
+			},
+			wantDelete: true,
+		},
+		{
+			name:         "delete failure",
+			ageThreshold: 0.3,
+			fillSecretData: func(t *testing.T, m map[string][]byte) {
+				caPEM, err := testutil.CreateCertificate(
+					time.Now().Add(-5*time.Hour),
+					time.Now().Add(5*time.Hour),
+				)
+				require.NoError(t, err)
+
+				// See cert_manager.go for this constant.
+				m["caCertificate"] = caPEM
+			},
+			configKubeAPIClient: func(c *kubernetesfake.Clientset) {
+				c.PrependReactor("delete", "secrets", func(_ kubetesting.Action) (bool, runtime.Object, error) {
+					return true, nil, errors.New("delete failed: some delete error")
+				})
+			},
+			wantError: "delete failed: some delete error",
+		},
+		{
+			name: "parse cert failure",
+			fillSecretData: func(t *testing.T, m map[string][]byte) {
+				privateKey, err := rsa.GenerateKey(rand.Reader, 2048)
+				require.NoError(t, err)
+
+				// See cert_manager.go for this constant.
+				m["caCertificate"] = x509.MarshalPKCS1PrivateKey(privateKey)
+			},
+			wantDelete: false,
+		},
+	}
+	for _, test := range tests {
+		test := test
+		t.Run(test.name, func(t *testing.T) {
+			t.Parallel()
+
+			ctx, cancel := context.WithTimeout(context.Background(), time.Second*3)
+			defer cancel()
+
+			kubeAPIClient := kubernetesfake.NewSimpleClientset()
+			if test.configKubeAPIClient != nil {
+				test.configKubeAPIClient(kubeAPIClient)
+			}
+
+			kubeInformerClient := kubernetesfake.NewSimpleClientset()
+			name := "api-serving-cert" // See cert_manager.go.
+			namespace := "some-namespace"
+			if test.fillSecretData != nil {
+				secret := &corev1.Secret{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      name,
+						Namespace: namespace,
+					},
+					Data: map[string][]byte{},
+				}
+				test.fillSecretData(t, secret.Data)
+
+				require.NoError(t, kubeAPIClient.Tracker().Add(secret))
+				require.NoError(t, kubeInformerClient.Tracker().Add(secret))
+			}
+
+			kubeInformers := kubeinformers.NewSharedInformerFactory(
+				kubeInformerClient,
+				0,
+			)
+
+			c := NewCertsExpirerController(
+				namespace,
+				kubeAPIClient,
+				kubeInformers.Core().V1().Secrets(),
+				controller.WithInformer,
+				test.ageThreshold,
+			)
+
+			// Must start informers before calling TestRunSynchronously().
+			kubeInformers.Start(ctx.Done())
+			controller.TestRunSynchronously(t, c)
+
+			err := controller.TestSync(t, c, controller.Context{
+				Context: ctx,
+			})
+			if test.wantError != "" {
+				require.EqualError(t, err, test.wantError)
+				return
+			}
+			require.NoError(t, err)
+
+			exActions := []kubetesting.Action{}
+			if test.wantDelete {
+				exActions = append(
+					exActions,
+					kubetesting.NewDeleteAction(
+						schema.GroupVersionResource{
+							Group:    "",
+							Version:  "v1",
+							Resource: "secrets",
+						},
+						namespace,
+						name,
+					),
+				)
+			}
+			acActions := kubeAPIClient.Actions()
+			require.Equal(t, exActions, acActions)
+		})
+	}
+}

internal/controller/apicerts/doc.go

@@ -0,0 +1,7 @@
+/*
+Copyright 2020 VMware, Inc.
+SPDX-License-Identifier: Apache-2.0
+*/
+
+// Package apicerts contains controllers that work together to provide rotating API certs.
+package apicerts

internal/controllermanager/prepare_controllers.go

@@ -34,6 +34,7 @@ func PrepareControllers(
 	serverInstallationNamespace string,
 	discoveryURLOverride *string,
 	dynamicCertProvider provider.DynamicTLSServingCertProvider,
+	servingCertRotationThreshold float32,
 ) (func(ctx context.Context), error) {
 	// Create k8s clients.
 	k8sClient, aggregatorClient, placeholderClient, err := createClients()
@@ -78,6 +79,16 @@ func PrepareControllers(
 				controller.WithInformer,
 			),
 			singletonWorker,
+		).
+		WithController(
+			apicerts.NewCertsExpirerController(
+				serverInstallationNamespace,
+				k8sClient,
+				installationNamespaceK8sInformers.Core().V1().Secrets(),
+				controller.WithInformer,
+				servingCertRotationThreshold,
+			),
+			singletonWorker,
 		)
 
 	// Return a function which starts the informers and controllers.

internal/server/server.go

@@ -10,9 +10,11 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"strconv"
 	"time"
 
 	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
 	genericapiserver "k8s.io/apiserver/pkg/server"
 	genericoptions "k8s.io/apiserver/pkg/server/options"
 	"k8s.io/apiserver/plugin/pkg/authenticator/token/webhook"
@@ -21,6 +23,7 @@ import (
 
 	"github.com/suzerain-io/placeholder-name/internal/apiserver"
 	"github.com/suzerain-io/placeholder-name/internal/certauthority/kubecertauthority"
+	"github.com/suzerain-io/placeholder-name/internal/constable"
 	"github.com/suzerain-io/placeholder-name/internal/controllermanager"
 	"github.com/suzerain-io/placeholder-name/internal/downward"
 	"github.com/suzerain-io/placeholder-name/internal/provider"
@@ -29,13 +32,39 @@ import (
 	"github.com/suzerain-io/placeholder-name/pkg/config"
 )
 
+type percentageValue struct {
+	percentage float32
+}
+
+var _ pflag.Value = &percentageValue{}
+
+func (p *percentageValue) String() string {
+	return fmt.Sprintf("%.2f%%", p.percentage*100)
+}
+
+func (p *percentageValue) Set(s string) error {
+	f, err := strconv.ParseFloat(s, 32)
+	if err != nil || f < 0 || f > 1 {
+		return constable.Error("must pass real number between 0 and 1")
+	}
+
+	p.percentage = float32(f)
+
+	return nil
+}
+
+func (p *percentageValue) Type() string {
+	return "percentage"
+}
+
 // App is an object that represents the placeholder-name-server application.
 type App struct {
 	cmd *cobra.Command
 
 	// CLI flags
-	configPath      string
+	configPath                   string
-	downwardAPIPath string
+	downwardAPIPath              string
+	servingCertRotationThreshold percentageValue
 }
 
 // This is ignored for now because we turn off etcd storage below, but this is
@@ -89,6 +118,13 @@ func addCommandlineFlagsToCommand(cmd *cobra.Command, app *App) {
 		"/etc/podinfo",
 		"path to Downward API volume mount",
 	)
+
+	app.servingCertRotationThreshold.percentage = .70 // default
+	cmd.Flags().Var(
+		&app.servingCertRotationThreshold,
+		"serving-cert-rotation-threshold",
+		"real number between 0 and 1 indicating percentage of lifetime before rotation of serving cert",
+	)
 }
 
 // Boot the aggregated API server, which will in turn boot the controllers.
@@ -132,6 +168,7 @@ func (a *App) runServer(ctx context.Context) error {
 		serverInstallationNamespace,
 		cfg.DiscoveryConfig.URL,
 		dynamicCertProvider,
+		a.servingCertRotationThreshold.percentage,
 	)
 	if err != nil {
 		return fmt.Errorf("could not prepare controllers: %w", err)

internal/server/server_test.go

@@ -25,10 +25,11 @@ Usage:
   placeholder-name-server [flags]
 
 Flags:
-  -c, --config string                  path to configuration file (default "placeholder-name.yaml")
+  -c, --config string                                path to configuration file (default "placeholder-name.yaml")
-      --downward-api-path string       path to Downward API volume mount (default "/etc/podinfo")
+      --downward-api-path string                     path to Downward API volume mount (default "/etc/podinfo")
-  -h, --help                           help for placeholder-name-server
+  -h, --help                                         help for placeholder-name-server
-      --log-flush-frequency duration   Maximum number of seconds between log flushes (default 5s)
+      --log-flush-frequency duration                 Maximum number of seconds between log flushes (default 5s)
+      --serving-cert-rotation-threshold percentage   real number between 0 and 1 indicating percentage of lifetime before rotation of serving cert (default 70.00%)
 `
 
 func TestCommand(t *testing.T) {
@@ -68,6 +69,30 @@ func TestCommand(t *testing.T) {
 			},
 			wantErr: `unknown command "tuna" for "placeholder-name-server"`,
 		},
+		{
+			name: "PercentageIsNotRealNumber",
+			args: []string{
+				"--config", "some/path/to/config.yaml",
+				"--serving-cert-rotation-threshold", "tuna",
+			},
+			wantErr: `invalid argument "tuna" for "--serving-cert-rotation-threshold" flag: must pass real number between 0 and 1`,
+		},
+		{
+			name: "PercentageIsTooSmall",
+			args: []string{
+				"--config", "some/path/to/config.yaml",
+				"--serving-cert-rotation-threshold", "-1",
+			},
+			wantErr: `invalid argument "-1" for "--serving-cert-rotation-threshold" flag: must pass real number between 0 and 1`,
+		},
+		{
+			name: "PercentageIsTooLarge",
+			args: []string{
+				"--config", "some/path/to/config.yaml",
+				"--serving-cert-rotation-threshold", "75",
+			},
+			wantErr: `invalid argument "75" for "--serving-cert-rotation-threshold" flag: must pass real number between 0 and 1`,
+		},
 	}
 	for _, test := range tests {
 		test := test

internal/testutil/certs.go

@@ -6,9 +6,13 @@ SPDX-License-Identifier: Apache-2.0
 package testutil
 
 import (
+	"crypto/rand"
+	"crypto/rsa"
 	"crypto/tls"
 	"crypto/x509"
+	"crypto/x509/pkix"
 	"encoding/pem"
+	"math/big"
 	"testing"
 	"time"
 
@@ -71,3 +75,40 @@ func (v *ValidCert) RequireMatchesPrivateKey(keyPEM string) {
 	_, err := tls.X509KeyPair([]byte(v.certPEM), []byte(keyPEM))
 	require.NoError(v.t, err)
 }
+
+// CreateCertificate creates a certificate with the provided time bounds, and
+// returns the PEM representation of the certificate.
+//
+// There is nothing very special about the certificate that it creates, just
+// that it is a valid certificate that can be used for testing.
+func CreateCertificate(notBefore, notAfter time.Time) ([]byte, error) {
+	privateKey, err := rsa.GenerateKey(rand.Reader, 2048)
+	if err != nil {
+		return nil, err
+	}
+
+	template := x509.Certificate{
+		SerialNumber: big.NewInt(0),
+		Subject: pkix.Name{
+			CommonName: "some-common-name",
+		},
+		NotBefore: notBefore,
+		NotAfter:  notAfter,
+	}
+	cert, err := x509.CreateCertificate(
+		rand.Reader,
+		&template,
+		&template,
+		&privateKey.PublicKey,
+		privateKey,
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	certPEM := pem.EncodeToMemory(&pem.Block{
+		Type:  "CERTIFICATE",
+		Bytes: cert,
+	})
+	return certPEM, nil
+}

test/go.mod

@@ -3,18 +3,27 @@ module github.com/suzerain-io/placeholder-name/test
 go 1.14
 
 require (
+	github.com/coreos/go-etcd v2.0.0+incompatible // indirect
+	github.com/cpuguy83/go-md2man v1.0.10 // indirect
 	github.com/davecgh/go-spew v1.1.1
+	github.com/docker/docker v0.7.3-0.20190327010347-be7ac8be2ae0 // indirect
+	github.com/evanphx/json-patch v4.2.0+incompatible // indirect
+	github.com/gophercloud/gophercloud v0.1.0 // indirect
 	github.com/stretchr/testify v1.6.1
+	github.com/suzerain-io/placeholder-name v0.0.0-20200819182107-1b9a70d089f4
 	github.com/suzerain-io/placeholder-name/kubernetes/1.19/api v0.0.0-00010101000000-000000000000
 	github.com/suzerain-io/placeholder-name/kubernetes/1.19/client-go v0.0.0-00010101000000-000000000000
 	github.com/suzerain-io/placeholder-name/pkg/client v0.0.0-00010101000000-000000000000
+	github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8 // indirect
 	k8s.io/api v0.19.0-rc.0
 	k8s.io/apimachinery v0.19.0-rc.0
 	k8s.io/client-go v0.19.0-rc.0
-	k8s.io/kube-aggregator v0.18.6
+	k8s.io/klog v1.0.0 // indirect
+	k8s.io/kube-aggregator v0.19.0-rc.0
 )
 
 replace (
+	github.com/suzerain-io/placeholder-name => ../
 	github.com/suzerain-io/placeholder-name/kubernetes/1.19/api => ../kubernetes/1.19/api
 	github.com/suzerain-io/placeholder-name/kubernetes/1.19/client-go => ../kubernetes/1.19/client-go
 	github.com/suzerain-io/placeholder-name/pkg/client => ../pkg/client

test/go.sum

@@ -41,13 +41,16 @@ github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdko
 github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
 github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
 github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
 github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
@@ -55,6 +58,7 @@ github.com/bombsimon/wsl/v3 v3.1.0 h1:E5SRssoBgtVFPcYWUOFJEcgaySgdtTNYzsSKDOY7ss
 github.com/bombsimon/wsl/v3 v3.1.0/go.mod h1:st10JtZYLE4D5sC7b8xV4zTKZwAQjCH/Hy2Pm1FNZIc=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
+github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
@@ -110,6 +114,7 @@ github.com/go-critic/go-critic v0.5.0/go.mod h1:4jeRh3ZAVnRYhuWdOEvwzVqLUpxMSoAT
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-lintpack/lintpack v0.5.2/go.mod h1:NwZuYi2nUHho8XEIZ6SIxihrnPoqBTDqfpXvXAN0sXM=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
@@ -165,10 +170,12 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfU
 github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7 h1:5ZkaAPbicIKTF2I64qf5Fh8Aa83Q/dnOafMYV0OMwjA=
 github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
+github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
 github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -196,6 +203,7 @@ github.com/golangci/gocyclo v0.0.0-20180528144436-0a533e8fa43d h1:pXTK/gkVNs7Zyy
 github.com/golangci/gocyclo v0.0.0-20180528144436-0a533e8fa43d/go.mod h1:ozx7R9SIwqmqf5pRP90DhR2Oay2UIjGuKheCBCNwAYU=
 github.com/golangci/gofmt v0.0.0-20190930125516-244bba706f1a h1:iR3fYXUjHCR97qWS8ch1y9zPNsgXThGwjKPrYfqMPks=
 github.com/golangci/gofmt v0.0.0-20190930125516-244bba706f1a/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU=
+github.com/golangci/golangci-lint v1.29.0/go.mod h1:Iq2GFBB9OoolSDWD81m0iJ2MR4MwDVbi4eC93fO7wh0=
 github.com/golangci/golangci-lint v1.30.0 h1:UhdK5WbO0GBd7W+k2lOD7BEJH4Wsa7zKfw8m3/aEJGQ=
 github.com/golangci/golangci-lint v1.30.0/go.mod h1:5t0i3wHlqQc9deBBvZsP+a/4xz7cfjV+zhp5U0Mzp14=
 github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc h1:gLLhTLMk2/SutryVJ6D4VZCU3CUqr8YloG7FPIBWFpI=
@@ -237,6 +245,7 @@ github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsC
 github.com/googleapis/gnostic v0.1.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
 github.com/googleapis/gnostic v0.4.1 h1:DLJCy1n/vrD4HPjOvYcT8aYQXpPIzoRZONaYwyycI+I=
 github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg=
+github.com/gookit/color v1.2.4/go.mod h1:AhIE+pS6D4Ql0SQWbBeXPHw7gY0/sjHoA4s/n1KB7xg=
 github.com/gookit/color v1.2.5/go.mod h1:AhIE+pS6D4Ql0SQWbBeXPHw7gY0/sjHoA4s/n1KB7xg=
 github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
@@ -266,6 +275,7 @@ github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/b
 github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
@@ -343,6 +353,7 @@ github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzp
 github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
 github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
 github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
 github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
@@ -355,6 +366,7 @@ github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0Qu
 github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -380,11 +392,13 @@ github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:v
 github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg=
 github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
 github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1lskyM0=
 github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
+github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA=
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
@@ -404,6 +418,7 @@ github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prY
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
+github.com/prometheus/client_golang v1.6.0/go.mod h1:ZLOG9ck3JLRdB5MgO8f+lLTe83AXG6ro35rLTxvnIl4=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
@@ -411,9 +426,11 @@ github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6T
 github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
 github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/procfs v0.0.11/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
 github.com/quasilyte/go-consistent v0.0.0-20190521200055-c6f3937de18c/go.mod h1:5STLWrekHfjyYwxBRVRXNOSewLJ3PWfDJd1VyTS21fI=
 github.com/quasilyte/go-ruleguard v0.1.2-0.20200318202121-b00d7a75d3d8 h1:DvnesvLtRPQOvaUbfXfh0tpMHg29by0H7F2U+QIkSu8=
@@ -431,7 +448,9 @@ github.com/ryanrolds/sqlclosecheck v0.3.0 h1:AZx+Bixh8zdUBxUA1NxbxVAS78vTPq4rCb8
 github.com/ryanrolds/sqlclosecheck v0.3.0/go.mod h1:1gREqxyTGR3lVtpngyFo3hZAgk0KCtEdgEkHwDbigdA=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
 github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw=
+github.com/sclevine/spec v1.4.0/go.mod h1:LvpgJaFyvQzRvc1kaDs0bulYwzC70PbiYjC4QnFHkOM=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
+github.com/securego/gosec/v2 v2.3.0/go.mod h1:UzeVyUXbxukhLeHKV3VVqo7HdoQR9MrRfFmZYotn8ME=
 github.com/securego/gosec/v2 v2.4.0 h1:ivAoWcY5DMs9n04Abc1VkqZBO0FL0h4ShTcVsC53lCE=
 github.com/securego/gosec/v2 v2.4.0/go.mod h1:0/Q4cjmlFDfDUj1+Fib61sc+U5IQb2w+Iv9/C3wPVko=
 github.com/shazow/go-diff v0.0.0-20160112020656-b6b7b6733b8c h1:W65qqJCIOVP4jpqPQ0YvHYKwcMEMVWIzWC5iNQQfBTU=
@@ -487,8 +506,13 @@ github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
+github.com/suzerain-io/controller-go v0.0.0-20200730212956-7f99b569ca9f h1:gZ6rAdl+VE9DT0yE52xY/kJZ/hOJYxwtsgGoPr5vItI=
+github.com/suzerain-io/controller-go v0.0.0-20200730212956-7f99b569ca9f/go.mod h1:+v9upryFWBJac6KXKlheGHr7e3kqpk1ldH1iIMFopMs=
+github.com/suzerain-io/placeholder-name v0.0.0-20200819182107-1b9a70d089f4 h1:3pdsX5UVieFJnYTtlNMoa/OHMzfHAfDwc91gT6LrtYU=
+github.com/suzerain-io/placeholder-name v0.0.0-20200819182107-1b9a70d089f4/go.mod h1:ohfv1ktuLQCoLviP17/IjYnDpbUOPkwsjH61P8Wraa8=
 github.com/tdakkota/asciicheck v0.0.0-20200416190851-d7f85be797a2 h1:Xr9gkxfOP0KQWXKNqmwe8vEeSUiUj4Rlee9CMVX2ZUQ=
 github.com/tdakkota/asciicheck v0.0.0-20200416190851-d7f85be797a2/go.mod h1:yHp0ai0Z9gUljN3o0xMhYJnH/IcvkdTBOX2fmJ93JEM=
+github.com/tetafro/godot v0.4.2/go.mod h1:/7NLHhv08H1+8DNj0MElpAACw1ajsCuf3TKNQxA5S+0=
 github.com/tetafro/godot v0.4.8 h1:h61+hQraWhdI6WYqMwAwZYCE5yxL6a9/Orw4REbabSU=
 github.com/tetafro/godot v0.4.8/go.mod h1:/7NLHhv08H1+8DNj0MElpAACw1ajsCuf3TKNQxA5S+0=
 github.com/timakin/bodyclose v0.0.0-20190930140734-f7f2e9bca95e h1:RumXZ56IrCj4CL+g1b9OL/oH0QnsF976bC8xQFYUD5Q=
@@ -517,7 +541,9 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
+go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
 go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg=
+go.etcd.io/etcd v0.5.0-alpha.5.0.20200520232829-54ba9589114f/go.mod h1:skWido08r9w6Lq/w70DO5XYIKMu4QFu1+4VsqLQuJy8=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
@@ -631,13 +657,18 @@ golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4 h1:5/PjkGUjvEU5Gl6BxmvKRPpqo2uNMv4rcHBMwzk/st8=
 golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
@@ -673,6 +704,7 @@ golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBn
 golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190719005602-e377ae9d6386/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI=
 golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
@@ -689,12 +721,15 @@ golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapK
 golang.org/x/tools v0.0.0-20200117220505-0cba7a3a9ee9/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200321224714-0d839f3cf2ed/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
 golang.org/x/tools v0.0.0-20200324003944-a576cf524670/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
+golang.org/x/tools v0.0.0-20200331202046-9d5940d49312/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20200414032229-332987a829c3/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20200422022333-3d57cf2e726e/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20200519015757-0d0afa43d58a/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200602230032-c00d67ef29d0/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20200625211823-6506e20df31f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20200626171337-aa94e735be7f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20200701041122-1837592efa10/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200710042808-f1c4188a97a1/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/tools v0.0.0-20200724022722-7017fd6b1305 h1:yaM5S0KcY0lIoZo7Fl+oi91b/DdlU2zuWpfHrpWbCS0=
 golang.org/x/tools v0.0.0-20200724022722-7017fd6b1305/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -764,6 +799,7 @@ gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bl
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
@@ -771,6 +807,7 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
+gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
@@ -785,14 +822,18 @@ k8s.io/apimachinery v0.18.6/go.mod h1:OaXp26zu/5J7p0f92ASynJa1pZo06YlV9fG7BoWbCk
 k8s.io/apimachinery v0.19.0-rc.0 h1:IBmRy0elJCGgxtCT0bHT93N+rhx+vF2DD1XXJ3ntLa8=
 k8s.io/apimachinery v0.19.0-rc.0/go.mod h1:EjWiYOPi+BZennZ5pGa3JLkQ+znhEOodGy/+umjiLDU=
 k8s.io/apiserver v0.18.6/go.mod h1:Zt2XvTHuaZjBz6EFYzpp+X4hTmgWGy8AthNVnTdm3Wg=
+k8s.io/apiserver v0.19.0-rc.0/go.mod h1:yEjU524zw/pxiG6nOsgY5Hu/akAg7tH/J/tKrLUp/mo=
 k8s.io/client-go v0.18.6/go.mod h1:/fwtGLjYMS1MaM5oi+eXhKwG+1UHidUEXRh6cNsdO0Q=
 k8s.io/client-go v0.19.0-rc.0 h1:6WW8MElhoLeYcLiN4ky1159XG5E39KYdmLCrV/6lNiE=
 k8s.io/client-go v0.19.0-rc.0/go.mod h1:3kWGD05F7c58atlk7ep9ob1hg2Yu9NSz8gJxCNNTHhc=
 k8s.io/code-generator v0.18.6/go.mod h1:TgNEVx9hCyPGpdtCWA34olQYLkh3ok9ar7XfSsr8b6c=
+k8s.io/code-generator v0.19.0-rc.0/go.mod h1:2jgaU9hVSqti1GiO69UFSoTZcL5XAvZSrXaNnK5RVA0=
 k8s.io/component-base v0.18.6/go.mod h1:knSVsibPR5K6EW2XOjEHik6sdU5nCvKMrzMt2D4In14=
+k8s.io/component-base v0.19.0-rc.0/go.mod h1:8cHxNUQdeDIIcORXOrMABUPbuEmbbHRtEweSSk8Il4g=
 k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
 k8s.io/gengo v0.0.0-20200114144118-36b2048a9120/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
 k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
+k8s.io/gengo v0.0.0-20200428234225-8167cfdcfc14/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
 k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
 k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
 k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8=
@@ -802,8 +843,11 @@ k8s.io/klog/v2 v2.2.0 h1:XRvcwJozkgZ1UQJmfMGpvRthQHOvihEhYtDfAaxMz/A=
 k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
 k8s.io/kube-aggregator v0.18.6 h1:xGP3oe0tAWEYnGWTnDPjXiIItekrnwDA2O7w0WqvGoo=
 k8s.io/kube-aggregator v0.18.6/go.mod h1:MKm8inLHdeiXQJCl6UdmgMosRrqJgyxO2obTXOkey/s=
+k8s.io/kube-aggregator v0.19.0-rc.0 h1:+u9y1c0R2GF8fuaEnlJrdUtxoEmQOON98oatycSquOA=
+k8s.io/kube-aggregator v0.19.0-rc.0/go.mod h1:DCq8Korz9XUEZVsq0wAGIAyJW79xdcYhIBtvWNTsTkc=
 k8s.io/kube-openapi v0.0.0-20200410145947-61e04a5be9a6/go.mod h1:GRQhZsXIAJ1xR0C9bd8UpWHZ5plfAS9fzPjJuQ6JL3E=
 k8s.io/kube-openapi v0.0.0-20200427153329-656914f816f9/go.mod h1:bfCVj+qXcEaE5SCvzBaqpOySr6tuCcpPKqF6HD8nyCw=
+k8s.io/kube-openapi v0.0.0-20200615155156-dffdd1682719 h1:n/ElZyI1dzFPXKS8nZMw8wozBUz7vEfL0Ja7jN2rSvA=
 k8s.io/kube-openapi v0.0.0-20200615155156-dffdd1682719/go.mod h1:bfCVj+qXcEaE5SCvzBaqpOySr6tuCcpPKqF6HD8nyCw=
 k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew=
 k8s.io/utils v0.0.0-20200619165400-6e3d28b6ed19 h1:7Nu2dTj82c6IaWvL7hImJzcXoTPz1MsSCH7r+0m6rfo=
@@ -817,7 +861,10 @@ mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b/go.mod h1:2odslEg/xrtNQqCYg2/jC
 mvdan.cc/unparam v0.0.0-20190720180237-d51796306d8f h1:Cq7MalBHYACRd6EesksG1Q8EoIAKOsiZviGKbOLIej4=
 mvdan.cc/unparam v0.0.0-20190720180237-d51796306d8f/go.mod h1:4G1h5nDURzA3bwVMZIVpwbkw+04kSxk3rAtzlimaUJw=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
+rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
+rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.7/go.mod h1:PHgbrJT7lCHcxMU+mDHEm+nx46H4zuuHZkDP6icnhu0=
+sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.9/go.mod h1:dzAXnQbTRyDlZPJX2SUPEqvnB+j7AJjtlox7PEwigU0=
 sigs.k8s.io/structured-merge-diff/v3 v3.0.0-20200116222232-67a7b8c61874/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw=
 sigs.k8s.io/structured-merge-diff/v3 v3.0.0 h1:dOmIZBMfhcHS09XZkMyUgkq5trg3/jRyJYFZUiaOp8E=
 sigs.k8s.io/structured-merge-diff/v3 v3.0.0/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw=

test/integration/api_serving_certs_test.go

@@ -13,85 +13,148 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
 
+	"github.com/suzerain-io/placeholder-name/internal/testutil"
 	"github.com/suzerain-io/placeholder-name/kubernetes/1.19/api/apis/placeholder/v1alpha1"
 	"github.com/suzerain-io/placeholder-name/test/library"
 )
 
 func TestAPIServingCertificateAutoCreationAndRotation(t *testing.T) {
 	library.SkipUnlessIntegration(t)
-	namespaceName := library.Getenv(t, "PLACEHOLDER_NAME_NAMESPACE")
 
-	kubeClient := library.NewClientset(t)
+	tests := []struct {
-	aggregatedClient := library.NewAggregatedClientset(t)
+		name          string
-	placeholderClient := library.NewPlaceholderNameClientset(t)
+		forceRotation func(context.Context, kubernetes.Interface, string) error
-	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+	}{
-	defer cancel()
+		{
-
+			name: "manual",
-	const apiServiceName = "v1alpha1.placeholder.suzerain-io.github.io"
+			forceRotation: func(
-
+				ctx context.Context,
-	// Get the initial auto-generated version of the Secret.
+				kubeClient kubernetes.Interface,
-	secret, err := kubeClient.CoreV1().Secrets(namespaceName).Get(ctx, "api-serving-cert", metav1.GetOptions{})
+				namespace string,
-	require.NoError(t, err)
+			) error {
-	initialCACert := secret.Data["caCertificate"]
+				// Delete the Secret, simulating an end user doing `kubectl delete` to manually ask for an immediate rotation.
-	initialPrivateKey := secret.Data["tlsPrivateKey"]
+				return kubeClient.
-	initialCertChain := secret.Data["tlsCertificateChain"]
+					CoreV1().
-	require.NotEmpty(t, initialCACert)
+					Secrets(namespace).
-	require.NotEmpty(t, initialPrivateKey)
+					Delete(ctx, "api-serving-cert", metav1.DeleteOptions{})
-	require.NotEmpty(t, initialCertChain)
-
-	// Check that the APIService has the same CA.
-	apiService, err := aggregatedClient.ApiregistrationV1().APIServices().Get(ctx, apiServiceName, metav1.GetOptions{})
-	require.NoError(t, err)
-	require.Equal(t, initialCACert, apiService.Spec.CABundle)
-
-	// Delete the Secret, simulating an end user doing `kubectl delete` to manually ask for an immediate rotation.
-	err = kubeClient.CoreV1().Secrets(namespaceName).Delete(ctx, "api-serving-cert", metav1.DeleteOptions{})
-	require.NoError(t, err)
-
-	// Expect that the Secret comes back right away with newly minted certs.
-	var secretIsRegenerated = func() bool {
-		secret, err = kubeClient.CoreV1().Secrets(namespaceName).Get(ctx, "api-serving-cert", metav1.GetOptions{})
-		return err == nil
-	}
-	assert.Eventually(t, secretIsRegenerated, 10*time.Second, 250*time.Millisecond)
-	require.NoError(t, err) // prints out the error in case of failure
-	regeneratedCACert := secret.Data["caCertificate"]
-	regeneratedPrivateKey := secret.Data["tlsPrivateKey"]
-	regeneratedCertChain := secret.Data["tlsCertificateChain"]
-	require.NotEmpty(t, regeneratedCACert)
-	require.NotEmpty(t, regeneratedPrivateKey)
-	require.NotEmpty(t, regeneratedCertChain)
-	require.NotEqual(t, initialCACert, regeneratedCACert)
-	require.NotEqual(t, initialPrivateKey, regeneratedPrivateKey)
-	require.NotEqual(t, initialCertChain, regeneratedCertChain)
-
-	// Expect that the APIService was also updated with the new CA.
-	var aggregatedAPIUpdated = func() bool {
-		apiService, err = aggregatedClient.ApiregistrationV1().APIServices().Get(ctx, apiServiceName, metav1.GetOptions{})
-		return err == nil
-	}
-	assert.Eventually(t, aggregatedAPIUpdated, 10*time.Second, 250*time.Millisecond)
-	require.NoError(t, err) // prints out the error in case of failure
-	require.Equal(t, regeneratedCACert, apiService.Spec.CABundle)
-
-	// Check that we can still make requests to the aggregated API through the kube API server,
-	// because the kube API server uses these certs when proxying requests to the aggregated API server,
-	// so this is effectively checking that the aggregated API server is using these new certs.
-	var aggregatedAPIWorking = func() bool {
-		_, err = placeholderClient.PlaceholderV1alpha1().CredentialRequests().Create(ctx, &v1alpha1.CredentialRequest{
-			TypeMeta:   metav1.TypeMeta{},
-			ObjectMeta: metav1.ObjectMeta{},
-			Spec: v1alpha1.CredentialRequestSpec{
-				Type:  v1alpha1.TokenCredentialType,
-				Token: &v1alpha1.CredentialRequestTokenCredential{Value: "not a good token"},
 			},
-		}, metav1.CreateOptions{})
+		},
-		// Should have got a success response with an error message inside it complaining about the token value.
+		{
-		return err == nil
+			name: "automatic",
+			forceRotation: func(
+				ctx context.Context,
+				kubeClient kubernetes.Interface,
+				namespace string,
+			) error {
+				// Create a cert that is expired - this should force the rotation controller
+				// to delete the cert, and therefore the cert should get rotated.
+				secret, err := kubeClient.
+					CoreV1().
+					Secrets(namespace).
+					Get(ctx, "api-serving-cert", metav1.GetOptions{})
+				if err != nil {
+					return err
+				}
+
+				secret.Data["caCertificate"], err = createExpiredCertificate()
+				if err != nil {
+					return err
+				}
+
+				_, err = kubeClient.
+					CoreV1().
+					Secrets(namespace).
+					Update(ctx, secret, metav1.UpdateOptions{})
+				return err
+			},
+		},
+	}
+	for _, test := range tests {
+		test := test
+		t.Run(test.name, func(t *testing.T) {
+			namespaceName := library.Getenv(t, "PLACEHOLDER_NAME_NAMESPACE")
+
+			kubeClient := library.NewClientset(t)
+			aggregatedClient := library.NewAggregatedClientset(t)
+			placeholderClient := library.NewPlaceholderNameClientset(t)
+			ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+			defer cancel()
+
+			const apiServiceName = "v1alpha1.placeholder.suzerain-io.github.io"
+
+			// Get the initial auto-generated version of the Secret.
+			secret, err := kubeClient.CoreV1().Secrets(namespaceName).Get(ctx, "api-serving-cert", metav1.GetOptions{})
+			require.NoError(t, err)
+			initialCACert := secret.Data["caCertificate"]
+			initialPrivateKey := secret.Data["tlsPrivateKey"]
+			initialCertChain := secret.Data["tlsCertificateChain"]
+			require.NotEmpty(t, initialCACert)
+			require.NotEmpty(t, initialPrivateKey)
+			require.NotEmpty(t, initialCertChain)
+
+			// Check that the APIService has the same CA.
+			apiService, err := aggregatedClient.ApiregistrationV1().APIServices().Get(ctx, apiServiceName, metav1.GetOptions{})
+			require.NoError(t, err)
+			require.Equal(t, initialCACert, apiService.Spec.CABundle)
+
+			// Force rotation to happen.
+			require.NoError(t, test.forceRotation(ctx, kubeClient, namespaceName))
+
+			// Expect that the Secret comes back right away with newly minted certs.
+			secretIsRegenerated := func() bool {
+				secret, err = kubeClient.CoreV1().Secrets(namespaceName).Get(ctx, "api-serving-cert", metav1.GetOptions{})
+				return err == nil
+			}
+			assert.Eventually(t, secretIsRegenerated, 10*time.Second, 250*time.Millisecond)
+			require.NoError(t, err) // prints out the error in case of failure
+			regeneratedCACert := secret.Data["caCertificate"]
+			regeneratedPrivateKey := secret.Data["tlsPrivateKey"]
+			regeneratedCertChain := secret.Data["tlsCertificateChain"]
+			require.NotEmpty(t, regeneratedCACert)
+			require.NotEmpty(t, regeneratedPrivateKey)
+			require.NotEmpty(t, regeneratedCertChain)
+			require.NotEqual(t, initialCACert, regeneratedCACert)
+			require.NotEqual(t, initialPrivateKey, regeneratedPrivateKey)
+			require.NotEqual(t, initialCertChain, regeneratedCertChain)
+
+			// Expect that the APIService was also updated with the new CA.
+			aggregatedAPIUpdated := func() bool {
+				apiService, err = aggregatedClient.ApiregistrationV1().APIServices().Get(ctx, apiServiceName, metav1.GetOptions{})
+				return err == nil
+			}
+			assert.Eventually(t, aggregatedAPIUpdated, 10*time.Second, 250*time.Millisecond)
+			require.NoError(t, err) // prints out the error in case of failure
+			require.Equal(t, regeneratedCACert, apiService.Spec.CABundle)
+
+			// Check that we can still make requests to the aggregated API through the kube API server,
+			// because the kube API server uses these certs when proxying requests to the aggregated API server,
+			// so this is effectively checking that the aggregated API server is using these new certs.
+			aggregatedAPIWorking := func() bool {
+				_, err = placeholderClient.PlaceholderV1alpha1().CredentialRequests().Create(ctx, &v1alpha1.CredentialRequest{
+					TypeMeta:   metav1.TypeMeta{},
+					ObjectMeta: metav1.ObjectMeta{},
+					Spec: v1alpha1.CredentialRequestSpec{
+						Type:  v1alpha1.TokenCredentialType,
+						Token: &v1alpha1.CredentialRequestTokenCredential{Value: "not a good token"},
+					},
+				}, metav1.CreateOptions{})
+				// Should have got a success response with an error message inside it complaining about the token value.
+				return err == nil
+			}
+
+			// Unfortunately, although our code changes all the certs immediately, it seems to take ~1 minute for
+			// the API machinery to notice that we updated our serving cert, causing 1 minute of downtime for our endpoint.
+			assert.Eventually(t, aggregatedAPIWorking, 2*time.Minute, 250*time.Millisecond)
+			require.NoError(t, err) // prints out the error in case of failure
+		})
 	}
-	// Unfortunately, although our code changes all the certs immediately, it seems to take ~1 minute for
+}
-	// the API machinery to notice that we updated our serving cert, causing 1 minute of downtime for our endpoint.
+
-	assert.Eventually(t, aggregatedAPIWorking, 2*time.Minute, 250*time.Millisecond)
+func createExpiredCertificate() ([]byte, error) {
-	require.NoError(t, err) // prints out the error in case of failure
+	return testutil.CreateCertificate(
+		time.Now().Add(-24*time.Hour),
+		time.Now().Add(-time.Hour),
+	)
 }