Switch back to an exec-based approach to grab the controller-manager CA. (#65)

This switches us back to an approach where we use the Pod "exec" API to grab the keys we need, rather than forcing our code to run on the control plane node. It will help us fail gracefully (or dynamically switch to alternate implementations) when the cluster is not self-hosted.

Signed-off-by: Matt Moyer <moyerm@vmware.com>
Co-authored-by: Ryan Richard <richardry@vmware.com>

deploy/deployment.yaml

@@ -46,13 +46,14 @@ data:
 #! TODO set up healthy, ready, etc. probes correctly?
 #! TODO set resource minimums (e.g. 512MB RAM) to make sure we get scheduled onto a reasonable node?
 apiVersion: apps/v1
-kind: DaemonSet
+kind: Deployment
 metadata:
   name: #@ data.values.app_name
   namespace: #@ data.values.namespace
   labels:
     app: #@ data.values.app_name
 spec:
+  replicas: 2
   selector:
     matchLabels:
       app: #@ data.values.app_name
@@ -79,15 +80,11 @@ spec:
           args:
             - --config=/etc/config/placeholder-name.yaml
             - --downward-api-path=/etc/podinfo
-            - --cluster-signing-cert-file=/etc/kubernetes/pki/ca.crt
-            - --cluster-signing-key-file=/etc/kubernetes/pki/ca.key
           volumeMounts:
             - name: config-volume
               mountPath: /etc/config
             - name: podinfo
               mountPath: /etc/podinfo
-            - name: k8s-certs
-              mountPath: /etc/kubernetes/pki
           livenessProbe:
             httpGet:
               path: /healthz
@@ -119,16 +116,10 @@ spec:
               - path: "namespace"
                 fieldRef:
                   fieldPath: metadata.namespace
-        - name: k8s-certs
-          hostPath:
-            path: /etc/kubernetes/pki
-            type: DirectoryOrCreate
-      nodeSelector: #! Create Pods on all nodes which match this node selector, and not on any other nodes.
-        node-role.kubernetes.io/master: ""
       tolerations:
         - key: CriticalAddonsOnly
           operator: Exists
-        - key: node-role.kubernetes.io/master #! Allow running on master nodes.
+        - key: node-role.kubernetes.io/master #! Allow running on master nodes too
           effect: NoSchedule
       #! "system-cluster-critical" cannot be used outside the kube-system namespace until Kubernetes >= 1.17,
       #! so we skip setting this for now (see https://github.com/kubernetes/kubernetes/issues/60596).

deploy/rbac.yaml

@@ -62,6 +62,35 @@ roleRef:
   name: #@ data.values.app_name + "-aggregated-api-server-role"
   apiGroup: rbac.authorization.k8s.io
 
+#! Give permission to list pods and pod exec in the kube-system namespace so we can find the API server's private key
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: #@ data.values.app_name + "-kube-system-pod-exec-role"
+  namespace: kube-system
+rules:
+  - apiGroups: [""]
+    resources: [pods]
+    verbs: [get, list]
+  - apiGroups: [""]
+    resources: [pods/exec]
+    verbs: [create]
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: #@ data.values.app_name + "-kube-system-pod-exec-role-binding"
+  namespace: kube-system
+subjects:
+  - kind: ServiceAccount
+    name: #@ data.values.app_name + "-service-account"
+    namespace: #@ data.values.namespace
+roleRef:
+  kind: Role
+  name: #@ data.values.app_name + "-kube-system-pod-exec-role"
+  apiGroup: rbac.authorization.k8s.io
+
 #! Allow both authenticated and unauthenticated CredentialRequests (i.e. allow all requests)
 ---
 apiVersion: rbac.authorization.k8s.io/v1

go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/google/go-cmp v0.5.0
 	github.com/sclevine/spec v1.4.0
 	github.com/spf13/cobra v1.0.0
+	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.6.1
 	github.com/suzerain-io/controller-go v0.0.0-20200730212956-7f99b569ca9f
 	github.com/suzerain-io/placeholder-name/kubernetes/1.19/api v0.0.0-00010101000000-000000000000

go.sum

@@ -100,6 +100,7 @@ github.com/denis-tingajkin/go-header v0.3.1/go.mod h1:sq/2IxMhaZX+RRcgHfCRx/m0M5
 github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
+github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96 h1:cenwrSVm+Z7QLSV/BsnenAOcDXdX4cMv4wP0B/5QbPg=
 github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
 github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
 github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=

internal/certauthority/certauthority.go

@@ -62,8 +62,8 @@ func secureEnv() env {
 var ErrInvalidCACertificate = fmt.Errorf("invalid CA certificate")
 
 // Load a certificate authority from an existing certificate and private key (in PEM format).
-func Load(certPath string, keyPath string) (*CA, error) {
+func Load(certPEM string, keyPEM string) (*CA, error) {
-	cert, err := tls.LoadX509KeyPair(certPath, keyPath)
+	cert, err := tls.X509KeyPair([]byte(certPEM), []byte(keyPEM))
 	if err != nil {
 		return nil, fmt.Errorf("could not load CA: %w", err)
 	}

internal/certauthority/certauthority_test.go

@@ -12,6 +12,7 @@ import (
 	"crypto/x509/pkix"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"strings"
 	"testing"
 	"time"
@@ -19,6 +20,19 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
+func loadFromFiles(t *testing.T, certPath string, keyPath string) (*CA, error) {
+	t.Helper()
+
+	certPEM, err := ioutil.ReadFile(certPath)
+	require.NoError(t, err)
+
+	keyPEM, err := ioutil.ReadFile(keyPath)
+	require.NoError(t, err)
+
+	ca, err := Load(string(certPEM), string(keyPEM))
+	return ca, err
+}
+
 func TestLoad(t *testing.T) {
 	tests := []struct {
 		name     string
@@ -26,30 +40,6 @@ func TestLoad(t *testing.T) {
 		keyPath  string
 		wantErr  string
 	}{
-		{
-			name:     "missing cert",
-			certPath: "./testdata/cert-does-not-exist",
-			keyPath:  "./testdata/test.key",
-			wantErr:  "could not load CA: open ./testdata/cert-does-not-exist: no such file or directory",
-		},
-		{
-			name:     "empty cert",
-			certPath: "./testdata/empty",
-			keyPath:  "./testdata/test.key",
-			wantErr:  "could not load CA: tls: failed to find any PEM data in certificate input",
-		},
-		{
-			name:     "invalid cert",
-			certPath: "./testdata/invalid",
-			keyPath:  "./testdata/test.key",
-			wantErr:  "could not load CA: tls: failed to find any PEM data in certificate input",
-		},
-		{
-			name:     "missing key",
-			certPath: "./testdata/test.crt",
-			keyPath:  "./testdata/key-does-not-exist",
-			wantErr:  "could not load CA: open ./testdata/key-does-not-exist: no such file or directory",
-		},
 		{
 			name:     "empty key",
 			certPath: "./testdata/test.crt",
@@ -83,7 +73,7 @@ func TestLoad(t *testing.T) {
 	for _, tt := range tests {
 		tt := tt
 		t.Run(tt.name, func(t *testing.T) {
-			ca, err := Load(tt.certPath, tt.keyPath)
+			ca, err := loadFromFiles(t, tt.certPath, tt.keyPath)
 			if tt.wantErr != "" {
 				require.EqualError(t, err, tt.wantErr)
 				return
@@ -202,7 +192,7 @@ func (e *errSigner) Sign(_ io.Reader, _ []byte, _ crypto.SignerOpts) ([]byte, er
 func TestIssue(t *testing.T) {
 	now := time.Date(2020, 7, 10, 12, 41, 12, 1234, time.UTC)
 
-	realCA, err := Load("./testdata/test.crt", "./testdata/test.key")
+	realCA, err := loadFromFiles(t, "./testdata/test.crt", "./testdata/test.key")
 	require.NoError(t, err)
 
 	tests := []struct {
@@ -302,7 +292,7 @@ func TestIssue(t *testing.T) {
 }
 
 func TestIssuePEM(t *testing.T) {
-	realCA, err := Load("./testdata/test.crt", "./testdata/test.key")
+	realCA, err := loadFromFiles(t, "./testdata/test.crt", "./testdata/test.key")
 	require.NoError(t, err)
 
 	certPEM, keyPEM, err := realCA.IssuePEM(pkix.Name{CommonName: "Test Server"}, []string{"example.com"}, 10*time.Minute)

internal/certauthority/kubecertauthority/kubecertauthority.go

@@ -0,0 +1,209 @@
+/*
+Copyright 2020 VMware, Inc.
+SPDX-License-Identifier: Apache-2.0
+*/
+
+// Package kubecertauthority implements a signer backed by the kubernetes controller-manager signing
+// keys (accessed via the kubernetes Exec API).
+package kubecertauthority
+
+import (
+	"bytes"
+	"context"
+	"crypto/x509/pkix"
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/spf13/pflag"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/deprecated/scheme"
+	"k8s.io/client-go/kubernetes"
+	restclient "k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/remotecommand"
+	"k8s.io/klog/v2"
+
+	"github.com/suzerain-io/placeholder-name/internal/certauthority"
+	"github.com/suzerain-io/placeholder-name/internal/constable"
+)
+
+// ErrNoKubeControllerManagerPod is returned when no kube-controller-manager pod is found on the cluster.
+const ErrNoKubeControllerManagerPod = constable.Error("did not find kube-controller-manager pod")
+
+const k8sAPIServerCACertPEMDefaultPath = "/etc/kubernetes/ca/ca.pem"
+const k8sAPIServerCAKeyPEMDefaultPath = "/etc/kubernetes/ca/ca.key"
+
+type signer interface {
+	IssuePEM(subject pkix.Name, dnsNames []string, ttl time.Duration) ([]byte, []byte, error)
+}
+
+type PodCommandExecutor interface {
+	Exec(podNamespace string, podName string, commandAndArgs ...string) (stdoutResult string, err error)
+}
+
+type kubeClientPodCommandExecutor struct {
+	kubeConfig *restclient.Config
+	kubeClient kubernetes.Interface
+}
+
+func NewPodCommandExecutor(kubeConfig *restclient.Config, kubeClient kubernetes.Interface) PodCommandExecutor {
+	return &kubeClientPodCommandExecutor{kubeConfig: kubeConfig, kubeClient: kubeClient}
+}
+
+func (s *kubeClientPodCommandExecutor) Exec(podNamespace string, podName string, commandAndArgs ...string) (string, error) {
+	request := s.kubeClient.
+		CoreV1().
+		RESTClient().
+		Post().
+		Namespace(podNamespace).
+		Resource("pods").
+		Name(podName).
+		SubResource("exec").
+		VersionedParams(&v1.PodExecOptions{
+			Stdin:   false,
+			Stdout:  true,
+			Stderr:  false,
+			TTY:     false,
+			Command: commandAndArgs,
+		}, scheme.ParameterCodec)
+
+	executor, err := remotecommand.NewSPDYExecutor(s.kubeConfig, "POST", request.URL())
+	if err != nil {
+		return "", err
+	}
+
+	var stdoutBuf bytes.Buffer
+	if err := executor.Stream(remotecommand.StreamOptions{Stdout: &stdoutBuf}); err != nil {
+		return "", err
+	}
+	return stdoutBuf.String(), nil
+}
+
+type CA struct {
+	kubeClient         kubernetes.Interface
+	podCommandExecutor PodCommandExecutor
+
+	shutdown, done chan struct{}
+
+	lock         sync.RWMutex
+	activeSigner signer
+}
+
+type ShutdownFunc func()
+
+// New creates a new instance of a CA which is has loaded the kube API server's private key
+// and is ready to issue certs, or an error. When successful, it also starts a goroutine
+// to periodically reload the kube API server's private key in case it changed, and returns
+// a function that can be used to shut down that goroutine.
+func New(kubeClient kubernetes.Interface, podCommandExecutor PodCommandExecutor, tick <-chan time.Time) (*CA, ShutdownFunc, error) {
+	signer, err := createSignerWithAPIServerSecret(kubeClient, podCommandExecutor)
+	if err != nil {
+		// The initial load failed, so give up
+		return nil, nil, err
+	}
+	result := &CA{
+		kubeClient:         kubeClient,
+		podCommandExecutor: podCommandExecutor,
+		activeSigner:       signer,
+		shutdown:           make(chan struct{}),
+		done:               make(chan struct{}),
+	}
+	go result.refreshLoop(tick)
+	return result, result.shutdownRefresh, nil
+}
+
+func createSignerWithAPIServerSecret(kubeClient kubernetes.Interface, podCommandExecutor PodCommandExecutor) (signer, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+	defer cancel()
+
+	pod, err := findControllerManagerPod(ctx, kubeClient)
+	if err != nil {
+		return nil, err
+	}
+	certPath, keyPath := getKeypairFilePaths(pod)
+
+	certPEM, err := podCommandExecutor.Exec(pod.Namespace, pod.Name, "cat", certPath)
+	if err != nil {
+		return nil, err
+	}
+
+	keyPEM, err := podCommandExecutor.Exec(pod.Namespace, pod.Name, "cat", keyPath)
+	if err != nil {
+		return nil, err
+	}
+
+	return certauthority.Load(certPEM, keyPEM)
+}
+
+func (c *CA) refreshLoop(tick <-chan time.Time) {
+	for {
+		select {
+		case <-c.shutdown:
+			close(c.done)
+			return
+		case <-tick:
+			c.updateSigner()
+		}
+	}
+}
+
+func (c *CA) updateSigner() {
+	newSigner, err := createSignerWithAPIServerSecret(c.kubeClient, c.podCommandExecutor)
+	if err != nil {
+		klog.Errorf("could not create signer with API server secret: %s", err)
+		return
+	}
+	c.lock.Lock()
+	c.activeSigner = newSigner
+	c.lock.Unlock()
+}
+
+func (c *CA) shutdownRefresh() {
+	close(c.shutdown)
+	<-c.done
+}
+
+// IssuePEM issues a new server certificate for the given identity and duration, returning it as a pair of
+//  PEM-formatted byte slices for the certificate and private key.
+func (c *CA) IssuePEM(subject pkix.Name, dnsNames []string, ttl time.Duration) ([]byte, []byte, error) {
+	c.lock.RLock()
+	signer := c.activeSigner
+	c.lock.RUnlock()
+
+	return signer.IssuePEM(subject, dnsNames, ttl)
+}
+
+func findControllerManagerPod(ctx context.Context, kubeClient kubernetes.Interface) (*v1.Pod, error) {
+	pods, err := kubeClient.CoreV1().Pods("kube-system").List(ctx, metav1.ListOptions{
+		LabelSelector: "component=kube-controller-manager",
+		FieldSelector: "status.phase=Running",
+	})
+	if err != nil {
+		return nil, fmt.Errorf("could not check for kube-controller-manager pod: %w", err)
+	}
+	for _, pod := range pods.Items {
+		return &pod, nil
+	}
+	return nil, ErrNoKubeControllerManagerPod
+}
+
+func getKeypairFilePaths(pod *v1.Pod) (string, string) {
+	certPath := getContainerArgByName(pod, "cluster-signing-cert-file", k8sAPIServerCACertPEMDefaultPath)
+	keyPath := getContainerArgByName(pod, "cluster-signing-key-file", k8sAPIServerCAKeyPEMDefaultPath)
+	return certPath, keyPath
+}
+
+func getContainerArgByName(pod *v1.Pod, name string, defaultValue string) string {
+	for _, container := range pod.Spec.Containers {
+		flagset := pflag.NewFlagSet("", pflag.ContinueOnError)
+		flagset.ParseErrorsWhitelist = pflag.ParseErrorsWhitelist{UnknownFlags: true}
+		var val string
+		flagset.StringVar(&val, name, "", "")
+		_ = flagset.Parse(append(container.Command, container.Args...))
+		if val != "" {
+			return val
+		}
+	}
+	return defaultValue
+}

internal/certauthority/kubecertauthority/kubecertauthority_test.go

@@ -0,0 +1,329 @@
+/*
+Copyright 2020 VMware, Inc.
+SPDX-License-Identifier: Apache-2.0
+*/
+
+package kubecertauthority
+
+import (
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"testing"
+	"time"
+
+	"github.com/sclevine/spec"
+	"github.com/sclevine/spec/report"
+	"github.com/stretchr/testify/require"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	kubernetesfake "k8s.io/client-go/kubernetes/fake"
+	"k8s.io/klog/v2"
+
+	"github.com/suzerain-io/placeholder-name/internal/testutil"
+)
+
+type fakePodExecutor struct {
+	resultsToReturn []string
+	errorsToReturn  []error
+
+	calledWithPodName        []string
+	calledWithPodNamespace   []string
+	calledWithCommandAndArgs [][]string
+
+	callCount int
+}
+
+func (s *fakePodExecutor) Exec(podNamespace string, podName string, commandAndArgs ...string) (string, error) {
+	s.calledWithPodNamespace = append(s.calledWithPodNamespace, podNamespace)
+	s.calledWithPodName = append(s.calledWithPodName, podName)
+	s.calledWithCommandAndArgs = append(s.calledWithCommandAndArgs, commandAndArgs)
+	result := s.resultsToReturn[s.callCount]
+	var err error = nil
+	if s.errorsToReturn != nil {
+		err = s.errorsToReturn[s.callCount]
+	}
+	s.callCount++
+	if err != nil {
+		return "", err
+	}
+	return result, nil
+}
+
+func TestCA(t *testing.T) {
+	spec.Run(t, "CA", func(t *testing.T, when spec.G, it spec.S) {
+		var r *require.Assertions
+		var fakeCertPEM, fakeKeyPEM string
+		var fakeCert2PEM, fakeKey2PEM string
+		var fakePod *corev1.Pod
+		var kubeAPIClient *kubernetesfake.Clientset
+		var fakeExecutor *fakePodExecutor
+		var neverTicker <-chan time.Time
+
+		var logger *testutil.TranscriptLogger
+
+		it.Before(func() {
+			r = require.New(t)
+
+			loadFile := func(filename string) string {
+				bytes, err := ioutil.ReadFile(filename)
+				r.NoError(err)
+				return string(bytes)
+			}
+			fakeCertPEM = loadFile("./testdata/test.crt")
+			fakeKeyPEM = loadFile("./testdata/test.key")
+			fakeCert2PEM = loadFile("./testdata/test2.crt")
+			fakeKey2PEM = loadFile("./testdata/test2.key")
+
+			fakePod = &corev1.Pod{
+				TypeMeta: metav1.TypeMeta{},
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "fake-pod",
+					Namespace: "kube-system",
+					Labels:    map[string]string{"component": "kube-controller-manager"},
+				},
+				Spec: corev1.PodSpec{
+					Containers: []corev1.Container{{Name: "kube-controller-manager"}},
+				},
+				Status: corev1.PodStatus{
+					Phase: "Running",
+				},
+			}
+
+			kubeAPIClient = kubernetesfake.NewSimpleClientset()
+
+			fakeExecutor = &fakePodExecutor{
+				resultsToReturn: []string{
+					fakeCertPEM,
+					fakeKeyPEM,
+					fakeCert2PEM,
+					fakeKey2PEM,
+				},
+			}
+
+			logger = testutil.NewTranscriptLogger(t)
+			klog.SetLogger(logger) // this is unfortunately a global logger, so can't run these tests in parallel :(
+		})
+
+		it.After(func() {
+			klog.SetLogger(nil)
+		})
+
+		when("the kube-controller-manager pod is found with default CLI flag values", func() {
+			it.Before(func() {
+				err := kubeAPIClient.Tracker().Add(fakePod)
+				r.NoError(err)
+			})
+
+			when("the exec commands return the API server's keypair", func() {
+				it("finds the API server's signing key and uses it to issue certificates", func() {
+					fakeTicker := make(chan time.Time)
+
+					subject, shutdownFunc, err := New(kubeAPIClient, fakeExecutor, fakeTicker)
+					r.NoError(err)
+					r.NotNil(shutdownFunc)
+					defer shutdownFunc()
+
+					r.Equal(2, fakeExecutor.callCount)
+
+					r.Equal("kube-system", fakeExecutor.calledWithPodNamespace[0])
+					r.Equal("fake-pod", fakeExecutor.calledWithPodName[0])
+					r.Equal([]string{"cat", "/etc/kubernetes/ca/ca.pem"}, fakeExecutor.calledWithCommandAndArgs[0])
+
+					r.Equal("kube-system", fakeExecutor.calledWithPodNamespace[1])
+					r.Equal("fake-pod", fakeExecutor.calledWithPodName[1])
+					r.Equal([]string{"cat", "/etc/kubernetes/ca/ca.key"}, fakeExecutor.calledWithCommandAndArgs[1])
+
+					// Validate that we can issue a certificate signed by the original API server CA.
+					certPEM, keyPEM, err := subject.IssuePEM(
+						pkix.Name{CommonName: "Test Server"},
+						[]string{"example.com"},
+						10*time.Minute,
+					)
+					r.NoError(err)
+					validCert := testutil.ValidateCertificate(t, fakeCertPEM, string(certPEM))
+					validCert.RequireDNSName("example.com")
+					validCert.RequireLifetime(time.Now(), time.Now().Add(10*time.Minute), 2*time.Minute)
+					validCert.RequireMatchesPrivateKey(string(keyPEM))
+
+					// Tick the timer and wait for another refresh loop to complete.
+					fakeTicker <- time.Now()
+
+					var secondCertPEM, secondKeyPEM string
+					r.Eventually(func() bool {
+						certPEM, keyPEM, err := subject.IssuePEM(
+							pkix.Name{CommonName: "Test Server"},
+							[]string{"example.com"},
+							10*time.Minute,
+						)
+						r.NoError(err)
+						secondCertPEM = string(certPEM)
+						secondKeyPEM = string(keyPEM)
+
+						block, _ := pem.Decode(certPEM)
+						require.NotNil(t, block)
+						parsed, err := x509.ParseCertificate(block.Bytes)
+						require.NoError(t, err)
+
+						// Validate the created cert using the second API server CA.
+						roots := x509.NewCertPool()
+						require.True(t, roots.AppendCertsFromPEM([]byte(fakeCert2PEM)))
+						opts := x509.VerifyOptions{Roots: roots}
+						_, err = parsed.Verify(opts)
+						return err == nil
+					}, 5*time.Second, 100*time.Millisecond)
+
+					validCert2 := testutil.ValidateCertificate(t, fakeCert2PEM, secondCertPEM)
+					validCert2.RequireDNSName("example.com")
+					validCert2.RequireLifetime(time.Now(), time.Now().Add(10*time.Minute), 2*time.Minute)
+					validCert2.RequireMatchesPrivateKey(secondKeyPEM)
+				})
+			})
+
+			when("the exec commands return the API server's keypair the first time but subsequently fails", func() {
+				it.Before(func() {
+					fakeExecutor.errorsToReturn = []error{nil, nil, fmt.Errorf("some exec error")}
+				})
+
+				it("logs an error message", func() {
+					fakeTicker := make(chan time.Time)
+
+					subject, shutdownFunc, err := New(kubeAPIClient, fakeExecutor, fakeTicker)
+					r.NoError(err)
+					r.NotNil(shutdownFunc)
+					defer shutdownFunc()
+					r.Equal(2, fakeExecutor.callCount)
+
+					// Tick the timer and wait for another refresh loop to complete.
+					fakeTicker <- time.Now()
+
+					// Wait for there to be a log output and require that it matches our expectation.
+					r.Eventually(func() bool { return len(logger.Transcript()) >= 1 }, 5*time.Second, 10*time.Millisecond)
+					r.Contains(logger.Transcript()[0].Message, "could not create signer with API server secret: some exec error")
+					r.Equal(logger.Transcript()[0].Level, "error")
+
+					// Validate that we can still issue a certificate signed by the original API server CA.
+					certPEM, _, err := subject.IssuePEM(
+						pkix.Name{CommonName: "Test Server"},
+						[]string{"example.com"},
+						10*time.Minute,
+					)
+					r.NoError(err)
+					testutil.ValidateCertificate(t, fakeCertPEM, string(certPEM))
+				})
+			})
+
+			when("the exec commands succeed but return garbage", func() {
+				it.Before(func() {
+					fakeExecutor.resultsToReturn = []string{"not a cert", "not a private key"}
+				})
+
+				it("returns an error", func() {
+					subject, shutdownFunc, err := New(kubeAPIClient, fakeExecutor, neverTicker)
+					r.Nil(subject)
+					r.Nil(shutdownFunc)
+					r.EqualError(err, "could not load CA: tls: failed to find any PEM data in certificate input")
+				})
+			})
+
+			when("the first exec command returns an error", func() {
+				it.Before(func() {
+					fakeExecutor.errorsToReturn = []error{fmt.Errorf("some error"), nil}
+				})
+
+				it("returns an error", func() {
+					subject, shutdownFunc, err := New(kubeAPIClient, fakeExecutor, neverTicker)
+					r.Nil(subject)
+					r.Nil(shutdownFunc)
+					r.EqualError(err, "some error")
+				})
+			})
+
+			when("the second exec command returns an error", func() {
+				it.Before(func() {
+					fakeExecutor.errorsToReturn = []error{nil, fmt.Errorf("some error")}
+				})
+
+				it("returns an error", func() {
+					subject, shutdownFunc, err := New(kubeAPIClient, fakeExecutor, neverTicker)
+					r.Nil(subject)
+					r.Nil(shutdownFunc)
+					r.EqualError(err, "some error")
+				})
+			})
+		})
+
+		when("the kube-controller-manager pod is found with non-default CLI flag values", func() {
+			it.Before(func() {
+				fakePod.Spec.Containers[0].Command = []string{
+					"kube-controller-manager",
+					"--cluster-signing-cert-file=/etc/kubernetes/ca/non-default.pem",
+				}
+				fakePod.Spec.Containers[0].Args = []string{
+					"--cluster-signing-key-file=/etc/kubernetes/ca/non-default.key",
+				}
+				err := kubeAPIClient.Tracker().Add(fakePod)
+				r.NoError(err)
+			})
+
+			it("finds the API server's signing key and uses it to issue certificates", func() {
+				_, shutdownFunc, err := New(kubeAPIClient, fakeExecutor, neverTicker)
+				r.NoError(err)
+				r.NotNil(shutdownFunc)
+				defer shutdownFunc()
+
+				r.Equal(2, fakeExecutor.callCount)
+
+				r.Equal("kube-system", fakeExecutor.calledWithPodNamespace[0])
+				r.Equal("fake-pod", fakeExecutor.calledWithPodName[0])
+				r.Equal([]string{"cat", "/etc/kubernetes/ca/non-default.pem"}, fakeExecutor.calledWithCommandAndArgs[0])
+
+				r.Equal("kube-system", fakeExecutor.calledWithPodNamespace[1])
+				r.Equal("fake-pod", fakeExecutor.calledWithPodName[1])
+				r.Equal([]string{"cat", "/etc/kubernetes/ca/non-default.key"}, fakeExecutor.calledWithCommandAndArgs[1])
+			})
+		})
+
+		when("the kube-controller-manager pod is found with non-default CLI flag values separated by spaces", func() {
+			it.Before(func() {
+				fakePod.Spec.Containers[0].Command = []string{
+					"kube-controller-manager",
+					"--cluster-signing-cert-file", "/etc/kubernetes/ca/non-default.pem",
+					"--cluster-signing-key-file", "/etc/kubernetes/ca/non-default.key",
+					"--foo=bar",
+				}
+				err := kubeAPIClient.Tracker().Add(fakePod)
+				r.NoError(err)
+			})
+
+			it("finds the API server's signing key and uses it to issue certificates", func() {
+				_, shutdownFunc, err := New(kubeAPIClient, fakeExecutor, neverTicker)
+				r.NoError(err)
+				r.NotNil(shutdownFunc)
+				defer shutdownFunc()
+
+				r.Equal(2, fakeExecutor.callCount)
+
+				r.Equal("kube-system", fakeExecutor.calledWithPodNamespace[0])
+				r.Equal("fake-pod", fakeExecutor.calledWithPodName[0])
+				r.Equal([]string{"cat", "/etc/kubernetes/ca/non-default.pem"}, fakeExecutor.calledWithCommandAndArgs[0])
+
+				r.Equal("kube-system", fakeExecutor.calledWithPodNamespace[1])
+				r.Equal("fake-pod", fakeExecutor.calledWithPodName[1])
+				r.Equal([]string{"cat", "/etc/kubernetes/ca/non-default.key"}, fakeExecutor.calledWithCommandAndArgs[1])
+			})
+		})
+
+		when("the kube-controller-manager pod is not found", func() {
+			it("returns an error", func() {
+				subject, shutdownFunc, err := New(kubeAPIClient, fakeExecutor, neverTicker)
+				r.Nil(subject)
+				r.Nil(shutdownFunc)
+				r.True(errors.Is(err, ErrNoKubeControllerManagerPod))
+			})
+		})
+	}, spec.Report(report.Terminal{}))
+}

internal/certauthority/kubecertauthority/testdata/test.crt

@@ -0,0 +1,17 @@
+-----BEGIN CERTIFICATE-----
+MIICyDCCAbCgAwIBAgIBADANBgkqhkiG9w0BAQsFADAVMRMwEQYDVQQDEwprdWJl
+cm5ldGVzMB4XDTIwMDcyNTIxMDQxOFoXDTMwMDcyMzIxMDQxOFowFTETMBEGA1UE
+AxMKa3ViZXJuZXRlczCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAL3K
+hYv2gIQ1Dwzh2cWMid+ofAnvLIfV2Xv61vTLGprUI+XUqB4/gtf6X6UNn0Lett2n
+d8p4wy7hw73hU/ggdvmWJvqBrSjc3JGfy+kj66fKXX+PTlbL7QbwiRvcSqIXIWlV
+lHHxECWrED8jCulw/NVqfook/h5iNUCT9yswSJr/0fImiVnoTlIoEYG2eCNejZ5c
+g39uD3ZTqd9ZxWwSLLnI+2kpJnZBPcd1ZQ8AQqzDgZtYRCqacn5gckQUKZWKQlxo
+Eft6g1XHJouAWAZw7hEtk0v8rG0/eKF7wamxFi6BFVlbjWBsB4T9rApbdBWTKeCJ
+Hv8fv5RMFSzpT3uzTO8CAwEAAaMjMCEwDgYDVR0PAQH/BAQDAgKkMA8GA1UdEwEB
+/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBACh5RhbxqJe+Z/gc17cZhKNmdiwu
+I2pLp3QBfwvN+Wbmajzw/7rYhY0d8JYVTJzXSCPWi6UAKxAtXOLF8WIIf9i39n6R
+uKOBGW14FzzGyRJiD3qaG/JTvEW+SLhwl68Ndr5LHSnbugAqq31abcQy6Zl9v5A8
+JKC97Lj/Sn8rj7opKy4W3oq7NCQsAb0zh4IllRF6UvSnJySfsg7xdXHHpxYDHtOS
+XcOu5ySUIZTgFe9RfeUZlGZ5xn0ckMlQ7qW2Wx1q0OVWw5us4NtkGqKrHG4Tn1X7
+uwo/Yytn5sDxrDv1/oii6AZOCsTPre4oD3wz4nmVzCVJcgrqH4Q24hT8WNg=
+-----END CERTIFICATE-----

internal/certauthority/kubecertauthority/testdata/test.key

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEogIBAAKCAQEAvcqFi/aAhDUPDOHZxYyJ36h8Ce8sh9XZe/rW9MsamtQj5dSo
+Hj+C1/pfpQ2fQt623ad3ynjDLuHDveFT+CB2+ZYm+oGtKNzckZ/L6SPrp8pdf49O
+VsvtBvCJG9xKohchaVWUcfEQJasQPyMK6XD81Wp+iiT+HmI1QJP3KzBImv/R8iaJ
+WehOUigRgbZ4I16NnlyDf24PdlOp31nFbBIsucj7aSkmdkE9x3VlDwBCrMOBm1hE
+KppyfmByRBQplYpCXGgR+3qDVccmi4BYBnDuES2TS/ysbT94oXvBqbEWLoEVWVuN
+YGwHhP2sClt0FZMp4Ike/x+/lEwVLOlPe7NM7wIDAQABAoIBAFC1tUEmHNUcM0BJ
+M3D9KQzB+63F1mwVlx1QOOV1EeVR3co5Ox1R6PSr9sycFGQ9jgqI0zp5TJe9Tp6L
+GkhklfPh1MWnK9o6wlnzWKXWrrp2Jni+mpPyuOPAmq4Maniv2XeP+0bROwqpyojv
+AA7yC7M+TH226ZJGNVs3EV9+cwHml0yuzBfIJn/rv/w2g+WRKM/MC0S7k2d8bRlA
+NycKVGAGBhKTltjoVYOeh6aHEpSjK8zfaePjo5dYJvoVIli60YCgcJOU/8jXT+Np
+1Fm7tRvAtj3pUp0Sqdaf2RUzh9jfJp2VFCHuSJ6TPqArOyQojtMcTHF0TiW7xrHP
+xOCRIAECgYEAwGBPU7vdthMJBg+ORUoGQQaItTeJvQwIqJvbKD2osp4jhS1dGZBw
+W30GKEc/gd8JNtOq9BBnMicPF7hktuy+bSPv41XPud67rSSO7Tsw20C10gFRq06B
+zIJWFAUqK3IkvVc3VDmtSLSDox4QZ/BdqaMlQ5y5JCsC5kThmkZFlO8CgYEA/I9X
+YHi6RioMJE1fqOHJL4DDjlezmcuRrD7fE5InKbtJZ2JhGYOX/C0KXnHTOWTCDxxN
+FBvpvD6Xv5o3PhB9Z6k2fqvJ4GS8urkG/KU4xcC+bak+9ava8oaiSqG16zD9NH2P
+jJ60NrbLl1J0pU9fiwuFVUKJ4hDZOfN9RqYdyAECgYAVwo8WhJiGgM6zfcz073OX
+pVqPTPHqjVLpZ3+5pIfRdGvGI6R1QM5EuvaYVb7MPOM47WZX5wcVOC/P2g6iVlMP
+21HGIC2384a9BfaYxOo40q/+SiHnw6CQ9mkwKIllkqqvNA9RGpkMMUb2i28For2l
+c4vCgxa6DZdtXns6TRqPxwKBgCfY5cxOv/T6BVhk7MbUeM2J31DB/ZAyUhV/Bess
+kAlBh19MYk2IOZ6L7KriApV3lDaWHIMjtEkDByYvyq98Io0MYZCywfMpca10K+oI
+l2B7/I+IuGpCZxUEsO5dfTpSTGDPvqpND9niFVUWqVi7oTNq6ep9yQtl5SADjqxq
+4SABAoGAIm0hUg1wtcS46cGLy6PIkPM5tocTSghtz4vFsuk/i4QA9GBoBO2gH6ty
++kJHmeaXt2dmgySp0QAWit5UlceEumB0NXnAdJZQxeGSFSyYkDWhwXd8wDceKo/1
+LfCU6Dk8IN/SsppVUWXQ2rlORvxlrHeCio8o0kS9Yiu55WMYg4g=
+-----END RSA PRIVATE KEY-----

internal/certauthority/kubecertauthority/testdata/test2.crt

@@ -0,0 +1,17 @@
+-----BEGIN CERTIFICATE-----
+MIICyDCCAbCgAwIBAgIBADANBgkqhkiG9w0BAQsFADAVMRMwEQYDVQQDEwprdWJl
+cm5ldGVzMB4XDTIwMDgxODE2NDEzNloXDTMwMDgxNjE2NDEzNlowFTETMBEGA1UE
+AxMKa3ViZXJuZXRlczCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALH7
+C2JpttDi3mxpD4bd+BZucCrS8XF2YwqYAr42HePp++PBnlUFqWmtPc9/bmo+7+7z
+iAAlnAV0pJWP+HR/PskX8MRcFAA1HoXLa37Q4SuBBQG+JE+AeaOObmQYaCFv55ej
+UF4+JIoQOdlbYEMYSI07el0cxQL4Io/CHJ3p7AtNElxjDuMK4B9W8NiCse3p7Uf+
+Qje4we1TYOfcpAM0jpBPHK9vCBCpX+j52S5DUTRVIk9kye3lCDmWOXH/fhj/aJTM
+1MP/hThbl2wIbFuv1bpa0kXNZs8xB63dtqROQ+lCghDmuayRmzwXl2PX6IgFFcjV
+yAgjXrZqjihs+mY8eT0CAwEAAaMjMCEwDgYDVR0PAQH/BAQDAgKkMA8GA1UdEwEB
+/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAE+Saqk2EyuIx1rxFWrOwpTi5q/B
+p/TwEtrmrFIRVPnGeBnhyfbGXPDMkzIY1mEvztu8H+pm5RPyhQYLsuwzYiYMQyxX
+yL9VvO7uydn7+3zX7oknQ5qAvN3nmItNyOKw3MRIKGsySNuTQ5JPtU/ufGlEivbK
+vNaDBqjKrBvwhIKMdV9/xYSyeBhSSWr/6W1tAk+XbHhQH1M78rdwGN5SI75L4FGu
+13kn/W2n8pE17TAY88B1YGKhsLSvf8KrFNYv+UUmzh2WstECKSlnbrSM+boMlGJn
+XahE8M23fieB+SaenQdOezrY4GAnXQ3qToDlhdYAOkWhcGDct47VRM93whY=
+-----END CERTIFICATE-----

internal/certauthority/kubecertauthority/testdata/test2.key

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEogIBAAKCAQEAsfsLYmm20OLebGkPht34Fm5wKtLxcXZjCpgCvjYd4+n748Ge
+VQWpaa09z39uaj7v7vOIACWcBXSklY/4dH8+yRfwxFwUADUehctrftDhK4EFAb4k
+T4B5o45uZBhoIW/nl6NQXj4kihA52VtgQxhIjTt6XRzFAvgij8IcnensC00SXGMO
+4wrgH1bw2IKx7entR/5CN7jB7VNg59ykAzSOkE8cr28IEKlf6PnZLkNRNFUiT2TJ
+7eUIOZY5cf9+GP9olMzUw/+FOFuXbAhsW6/VulrSRc1mzzEHrd22pE5D6UKCEOa5
+rJGbPBeXY9foiAUVyNXICCNetmqOKGz6Zjx5PQIDAQABAoIBAD06klYO7De8dKxz
+EEZjgn+lCq2Q2EMiaTwxw2/QikPoMSHPcDrrsbaLROJngoLGmCBqY3U5ew1dbWmO
+l/jr9ZuUwt2ql67il1eL/bUpAu3GewR4d2FqX25nB48j3l7ycof2RSXG1ycwIdam
+2tz6M6tytMvno9c7qhguvU2ONghEreXG3YYLdf9l97aB+p6GdXhwty22b7tAVwp1
+GKn79kVYgmL86lph9hBPqtHuG1LHZUiFodr2iWXSu3H/265OD58a33ZO3iyfFI0s
+PPy87ZN0r+1hGpoKKkDe63udOYgAG6xmIea/1Pdn9Eg87tueoeC7XcUpdaCJlKaF
+tqCusEECgYEA60rWyXxTFKJ4QdVaqXoWMA4cQkT73RxznSKwkN/Svk8TVv+p5s5Y
+oYKN4qyMzxvQzu+QNWpd1yTveCmmEynz457ELpGtidtiJdm7xZMdMGrU02eCL9mZ
+ERbtfAkbEAKvN8D73fWyzghKv4dgcQptmsqZlYYc4vpwHveK+/N5lukCgYEAwaT3
+iMTWCv7Vp87iKrzNUAH4iBWlazwbE+EDEnHVw26Y82fhgEgxiU2UvFSaIVhGpaCz
+MYSXSdRcQTHgCoJLPfWHUHTJPqf36KfAJfdaxxjzTTbNYjUOkdcUD1bcNrm0yjoY
+nR4zK1FPw86ODMYtBpfkyL7ZX8G1v5pRL/6/gzUCgYBzgwQ7Wmu3H6QGPeYKecNW
+yDabWh6ECKnBpPwlw5xEjbGi7lTM2NSuRde+RpPCQZebYATeFGAJdTqTNW8wzVHM
+l28cpawal7dxeZkzf+u+j1P4jUJel2cL+sOQNzAwBgFbT8TWzP6BI5T+vklcdZAl
+g/0uaO7Zh7Vvnnt/AaLZsQKBgGfbHzuGPjoFdPecOKatPfxUIkRyP5bk1KzzuF8T
+GI/JaFTbeREBJzg5mLTtNwD9RF6ecpzzPOTG9Xet1Tgtq0cewSUAjdKB6a8pESAL
+qu8vTYYzBzJNvHOxg7u6XT8omHMBd6QEx3LLGFmvFXZ6bzmjC3wzB4iY7u5FSJfS
+LEqlAoGAb0rbJ85vrJopbx8lzhJjyaJfM8+A3oQg1K3TphHrcgkUc8qx8QEosziM
+wzYKSBlXd2bxMibyd0mTEMNl4/BqofaKoqof9gBIbkamwXOO8s7IgKxQAfr1R/z8
+tHBW/g0QWPB+qtaVDtHwyQLlxjx8HD7atIo8d/do9ruwVaf+r6g=
+-----END RSA PRIVATE KEY-----

internal/controller/apicerts/certs_manager_test.go

@@ -7,9 +7,6 @@ package apicerts
 
 import (
 	"context"
-	"crypto/tls"
-	"crypto/x509"
-	"encoding/pem"
 	"errors"
 	"testing"
 	"time"
@@ -222,29 +219,10 @@ func TestManagerControllerSync(t *testing.T) {
 					r.NotEmpty(actualCertChain)
 
 					// Validate the created cert using the CA, and also validate the cert's hostname
-					roots := x509.NewCertPool()
+					validCert := testutil.ValidateCertificate(t, actualCACert, actualCertChain)
-					ok := roots.AppendCertsFromPEM([]byte(actualCACert))
+					validCert.RequireDNSName("placeholder-name-api." + installedInNamespace + ".svc")
-					r.True(ok)
+					validCert.RequireLifetime(time.Now(), time.Now().Add(24*365*time.Hour), 2*time.Minute)
-					block, _ := pem.Decode([]byte(actualCertChain))
+					validCert.RequireMatchesPrivateKey(actualPrivateKey)
-					r.NotNil(block)
-					parsedCert, err := x509.ParseCertificate(block.Bytes)
-					r.NoError(err)
-					serviceEndpoint := "placeholder-name-api." + installedInNamespace + ".svc"
-					opts := x509.VerifyOptions{
-						DNSName: serviceEndpoint,
-						Roots:   roots,
-					}
-					_, err = parsedCert.Verify(opts)
-					r.NoError(err)
-					r.Contains(parsedCert.DNSNames, serviceEndpoint, "expected an explicit DNS SAN, not just Common Name")
-
-					// Check the created cert's validity bounds
-					r.WithinDuration(time.Now(), parsedCert.NotBefore, time.Minute*2)
-					r.WithinDuration(time.Now().Add(24*365*time.Hour), parsedCert.NotAfter, time.Minute*2)
-
-					// Check that the private key and cert chain match
-					_, err = tls.X509KeyPair([]byte(actualCertChain), []byte(actualPrivateKey))
-					r.NoError(err)
 
 					// Make sure we updated the APIService caBundle and left it otherwise unchanged
 					r.Len(aggregatorAPIClient.Actions(), 2)

internal/controllermanager/prepare_controllers.go

@@ -97,7 +97,7 @@ func createClients() (
 	placeholderClient *placeholderclientset.Clientset,
 	err error,
 ) {
-	// Load the Kubernetes client configuration (kubeconfig),
+	// Load the Kubernetes client configuration.
 	kubeConfig, err := restclient.InClusterConfig()
 	if err != nil {
 		return nil, nil, nil, fmt.Errorf("could not load in-cluster configuration: %w", err)

internal/registry/credentialrequest/rest_test.go

@@ -425,9 +425,10 @@ func TestCreate(t *testing.T) {
 }
 
 func requireOneLogStatement(r *require.Assertions, logger *testutil.TranscriptLogger, messageContains string) {
-	r.Len(logger.Transcript, 1)
+	transcript := logger.Transcript()
-	r.Equal("info", logger.Transcript[0].Level)
+	r.Len(transcript, 1)
-	r.Contains(logger.Transcript[0].Message, messageContains)
+	r.Equal("info", transcript[0].Level)
+	r.Contains(transcript[0].Message, messageContains)
 }
 
 func callCreate(ctx context.Context, storage *REST, credentialRequest *placeholderapi.CredentialRequest) (runtime.Object, error) {

internal/server/server.go

@@ -10,17 +10,21 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"time"
 
 	"github.com/spf13/cobra"
 	genericapiserver "k8s.io/apiserver/pkg/server"
 	genericoptions "k8s.io/apiserver/pkg/server/options"
 	"k8s.io/apiserver/plugin/pkg/authenticator/token/webhook"
+	"k8s.io/client-go/kubernetes"
+	restclient "k8s.io/client-go/rest"
 
 	"github.com/suzerain-io/placeholder-name/internal/apiserver"
-	"github.com/suzerain-io/placeholder-name/internal/certauthority"
+	"github.com/suzerain-io/placeholder-name/internal/certauthority/kubecertauthority"
 	"github.com/suzerain-io/placeholder-name/internal/controllermanager"
 	"github.com/suzerain-io/placeholder-name/internal/downward"
 	"github.com/suzerain-io/placeholder-name/internal/provider"
+	"github.com/suzerain-io/placeholder-name/internal/registry/credentialrequest"
 	placeholderv1alpha1 "github.com/suzerain-io/placeholder-name/kubernetes/1.19/api/apis/placeholder/v1alpha1"
 	"github.com/suzerain-io/placeholder-name/pkg/config"
 )
@@ -30,10 +34,8 @@ type App struct {
 	cmd *cobra.Command
 
 	// CLI flags
-	configPath                 string
+	configPath      string
-	downwardAPIPath            string
+	downwardAPIPath string
-	clusterSigningCertFilePath string
-	clusterSigningKeyFilePath  string
 }
 
 // This is ignored for now because we turn off etcd storage below, but this is
@@ -87,20 +89,6 @@ func addCommandlineFlagsToCommand(cmd *cobra.Command, app *App) {
 		"/etc/podinfo",
 		"path to Downward API volume mount",
 	)
-
-	cmd.Flags().StringVar(
-		&app.clusterSigningCertFilePath,
-		"cluster-signing-cert-file",
-		"",
-		"path to cluster signing certificate",
-	)
-
-	cmd.Flags().StringVar(
-		&app.clusterSigningKeyFilePath,
-		"cluster-signing-key-file",
-		"",
-		"path to cluster signing private key",
-	)
 }
 
 // Boot the aggregated API server, which will in turn boot the controllers.
@@ -112,10 +100,11 @@ func (a *App) runServer(ctx context.Context) error {
 	}
 
 	// Load the Kubernetes cluster signing CA.
-	k8sClusterCA, err := certauthority.Load(a.clusterSigningCertFilePath, a.clusterSigningKeyFilePath)
+	k8sClusterCA, shutdownCA, err := getClusterCASigner()
 	if err != nil {
-		return fmt.Errorf("could not load cluster signing CA: %w", err)
+		return err
 	}
+	defer shutdownCA()
 
 	// Create a WebhookTokenAuthenticator.
 	webhookTokenAuthenticator, err := config.NewWebhook(cfg.WebhookConfig)
@@ -169,11 +158,40 @@ func (a *App) runServer(ctx context.Context) error {
 	return server.GenericAPIServer.PrepareRun().Run(ctx.Done())
 }
 
+func getClusterCASigner() (*kubecertauthority.CA, kubecertauthority.ShutdownFunc, error) {
+	// Load the Kubernetes client configuration.
+	kubeConfig, err := restclient.InClusterConfig()
+	if err != nil {
+		return nil, nil, fmt.Errorf("could not load in-cluster configuration: %w", err)
+	}
+
+	// Connect to the core Kubernetes API.
+	kubeClient, err := kubernetes.NewForConfig(kubeConfig)
+	if err != nil {
+		return nil, nil, fmt.Errorf("could not initialize Kubernetes client: %w", err)
+	}
+
+	// Make a clock tick that triggers a periodic refresh.
+	ticker := time.NewTicker(5 * time.Minute)
+
+	// Make a CA which uses the Kubernetes cluster API server's signing certs.
+	k8sClusterCA, shutdownCA, err := kubecertauthority.New(
+		kubeClient,
+		kubecertauthority.NewPodCommandExecutor(kubeConfig, kubeClient),
+		ticker.C,
+	)
+	if err != nil {
+		ticker.Stop()
+		return nil, nil, fmt.Errorf("could not load cluster signing CA: %w", err)
+	}
+	return k8sClusterCA, func() { shutdownCA(); ticker.Stop() }, nil
+}
+
 // Create a configuration for the aggregated API server.
 func getAggregatedAPIServerConfig(
 	dynamicCertProvider provider.DynamicTLSServingCertProvider,
 	webhookTokenAuthenticator *webhook.WebhookTokenAuthenticator,
-	ca *certauthority.CA,
+	ca credentialrequest.CertIssuer,
 	startControllersPostStartHook func(context.Context),
 ) (*apiserver.Config, error) {
 	recommendedOptions := genericoptions.NewRecommendedOptions(

internal/server/server_test.go

@@ -25,12 +25,10 @@ Usage:
   placeholder-name-server [flags]
 
 Flags:
-      --cluster-signing-cert-file string   path to cluster signing certificate
+  -c, --config string                  path to configuration file (default "placeholder-name.yaml")
-      --cluster-signing-key-file string    path to cluster signing private key
+      --downward-api-path string       path to Downward API volume mount (default "/etc/podinfo")
-  -c, --config string                      path to configuration file (default "placeholder-name.yaml")
+  -h, --help                           help for placeholder-name-server
-      --downward-api-path string           path to Downward API volume mount (default "/etc/podinfo")
+      --log-flush-frequency duration   Maximum number of seconds between log flushes (default 5s)
-  -h, --help                               help for placeholder-name-server
-      --log-flush-frequency duration       Maximum number of seconds between log flushes (default 5s)
 `
 
 func TestCommand(t *testing.T) {

internal/testutil/certs.go

@@ -0,0 +1,73 @@
+/*
+Copyright 2020 VMware, Inc.
+SPDX-License-Identifier: Apache-2.0
+*/
+
+package testutil
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/pem"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+type ValidCert struct {
+	t       *testing.T
+	roots   *x509.CertPool
+	certPEM string
+	parsed  *x509.Certificate
+}
+
+// ValidateCertificate validates a certificate and provides an object for asserting properties of the certificate.
+func ValidateCertificate(t *testing.T, caPEM string, certPEM string) *ValidCert {
+	t.Helper()
+
+	block, _ := pem.Decode([]byte(certPEM))
+	require.NotNil(t, block)
+	parsed, err := x509.ParseCertificate(block.Bytes)
+	require.NoError(t, err)
+
+	// Validate the created cert using the CA.
+	roots := x509.NewCertPool()
+	require.True(t, roots.AppendCertsFromPEM([]byte(caPEM)))
+	opts := x509.VerifyOptions{Roots: roots}
+	_, err = parsed.Verify(opts)
+	require.NoError(t, err)
+
+	return &ValidCert{
+		t:       t,
+		roots:   roots,
+		certPEM: certPEM,
+		parsed:  parsed,
+	}
+}
+
+// RequireDNSName asserts that the certificate matches the provided DNS name.
+func (v *ValidCert) RequireDNSName(expectDNSName string) {
+	v.t.Helper()
+	opts := x509.VerifyOptions{
+		Roots:   v.roots,
+		DNSName: expectDNSName,
+	}
+	_, err := v.parsed.Verify(opts)
+	require.NoError(v.t, err)
+	require.Contains(v.t, v.parsed.DNSNames, expectDNSName, "expected an explicit DNS SAN, not just Common Name")
+}
+
+// RequireLifetime asserts that the lifetime of the certificate matches the expected timestamps.
+func (v *ValidCert) RequireLifetime(expectNotBefore time.Time, expectNotAfter time.Time, delta time.Duration) {
+	v.t.Helper()
+	require.WithinDuration(v.t, expectNotBefore, v.parsed.NotBefore, delta)
+	require.WithinDuration(v.t, expectNotAfter, v.parsed.NotAfter, delta)
+}
+
+// RequireMatchesPrivateKey asserts that the public key in the certificate matches the provided private key.
+func (v *ValidCert) RequireMatchesPrivateKey(keyPEM string) {
+	v.t.Helper()
+	_, err := tls.X509KeyPair([]byte(v.certPEM), []byte(keyPEM))
+	require.NoError(v.t, err)
+}

internal/testutil/transcript_logger.go

@@ -7,6 +7,7 @@ package testutil
 
 import (
 	"fmt"
+	"sync"
 	"testing"
 
 	"github.com/go-logr/logr"
@@ -14,7 +15,8 @@ import (
 
 type TranscriptLogger struct {
 	t          *testing.T
-	Transcript []TranscriptLogMessage
+	lock       sync.Mutex
+	transcript []TranscriptLogMessage
 }
 
 var _ logr.Logger = &TranscriptLogger{}
@@ -28,15 +30,27 @@ func NewTranscriptLogger(t *testing.T) *TranscriptLogger {
 	return &TranscriptLogger{t: t}
 }
 
+func (log *TranscriptLogger) Transcript() []TranscriptLogMessage {
+	log.lock.Lock()
+	defer log.lock.Unlock()
+	result := make([]TranscriptLogMessage, 0, len(log.transcript))
+	result = append(result, log.transcript...)
+	return result
+}
+
 func (log *TranscriptLogger) Info(msg string, keysAndValues ...interface{}) {
-	log.Transcript = append(log.Transcript, TranscriptLogMessage{
+	log.lock.Lock()
+	defer log.lock.Unlock()
+	log.transcript = append(log.transcript, TranscriptLogMessage{
 		Level:   "info",
 		Message: fmt.Sprintf(msg, keysAndValues...),
 	})
 }
 
 func (log *TranscriptLogger) Error(err error, msg string, keysAndValues ...interface{}) {
-	log.Transcript = append(log.Transcript, TranscriptLogMessage{
+	log.lock.Lock()
+	defer log.lock.Unlock()
+	log.transcript = append(log.transcript, TranscriptLogMessage{
 		Level:   "error",
 		Message: fmt.Sprintf("%s: %v -- %v", msg, err, keysAndValues),
 	})

test/integration/app_availability_test.go

@@ -11,23 +11,39 @@ import (
 	"time"
 
 	"github.com/stretchr/testify/require"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	"github.com/suzerain-io/placeholder-name/test/library"
 )
 
-func TestAppAvailability(t *testing.T) {
+func TestGetDeployment(t *testing.T) {
 	library.SkipUnlessIntegration(t)
 	namespaceName := library.Getenv(t, "PLACEHOLDER_NAME_NAMESPACE")
-	daemonSetName := library.Getenv(t, "PLACEHOLDER_NAME_APP_NAME")
+	deploymentName := library.Getenv(t, "PLACEHOLDER_NAME_APP_NAME")
 
 	client := library.NewClientset(t)
 
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
 
-	daemonSet, err := client.AppsV1().DaemonSets(namespaceName).Get(ctx, daemonSetName, metav1.GetOptions{})
+	appDeployment, err := client.AppsV1().Deployments(namespaceName).Get(ctx, deploymentName, metav1.GetOptions{})
 	require.NoError(t, err)
 
-	require.GreaterOrEqual(t, daemonSet.Status.NumberAvailable, int32(1))
+	cond := getDeploymentCondition(appDeployment.Status, appsv1.DeploymentAvailable)
+	require.NotNil(t, cond)
+	require.Equalf(t, corev1.ConditionTrue, cond.Status, "app should be available: %s", library.Sdump(appDeployment))
+}
+
+// getDeploymentCondition returns the condition with the provided type.
+// Copied from k8s.io/kubectl/pkg/util/deployment/deployment.go to prevent us from vendoring the world.
+func getDeploymentCondition(status appsv1.DeploymentStatus, condType appsv1.DeploymentConditionType) *appsv1.DeploymentCondition {
+	for i := range status.Conditions {
+		c := status.Conditions[i]
+		if c.Type == condType {
+			return &c
+		}
+	}
+	return nil
 }