From 1b9a70d089f41862b2ec258458e48dcf7d715fab Mon Sep 17 00:00:00 2001 From: Matt Moyer Date: Wed, 19 Aug 2020 13:21:07 -0500 Subject: [PATCH] Switch back to an exec-based approach to grab the controller-manager CA. (#65) This switches us back to an approach where we use the Pod "exec" API to grab the keys we need, rather than forcing our code to run on the control plane node. It will help us fail gracefully (or dynamically switch to alternate implementations) when the cluster is not self-hosted. Signed-off-by: Matt Moyer Co-authored-by: Ryan Richard --- deploy/deployment.yaml | 15 +- deploy/rbac.yaml | 29 ++ go.mod | 1 + go.sum | 1 + internal/certauthority/certauthority.go | 4 +- internal/certauthority/certauthority_test.go | 44 +-- .../kubecertauthority/kubecertauthority.go | 209 +++++++++++ .../kubecertauthority_test.go | 329 ++++++++++++++++++ .../kubecertauthority/testdata/test.crt | 17 + .../kubecertauthority/testdata/test.key | 27 ++ .../kubecertauthority/testdata/test2.crt | 17 + .../kubecertauthority/testdata/test2.key | 27 ++ .../controller/apicerts/certs_manager_test.go | 30 +- .../controllermanager/prepare_controllers.go | 2 +- .../registry/credentialrequest/rest_test.go | 7 +- internal/server/server.go | 62 ++-- internal/server/server_test.go | 10 +- internal/testutil/certs.go | 73 ++++ internal/testutil/transcript_logger.go | 20 +- test/integration/app_availability_test.go | 24 +- 20 files changed, 842 insertions(+), 106 deletions(-) create mode 100644 internal/certauthority/kubecertauthority/kubecertauthority.go create mode 100644 internal/certauthority/kubecertauthority/kubecertauthority_test.go create mode 100644 internal/certauthority/kubecertauthority/testdata/test.crt create mode 100644 internal/certauthority/kubecertauthority/testdata/test.key create mode 100644 internal/certauthority/kubecertauthority/testdata/test2.crt create mode 100644 internal/certauthority/kubecertauthority/testdata/test2.key create mode 100644 internal/testutil/certs.go diff --git a/deploy/deployment.yaml b/deploy/deployment.yaml index 48c9b8d8..5f752f46 100644 --- a/deploy/deployment.yaml +++ b/deploy/deployment.yaml @@ -46,13 +46,14 @@ data: #! TODO set up healthy, ready, etc. probes correctly? #! TODO set resource minimums (e.g. 512MB RAM) to make sure we get scheduled onto a reasonable node? apiVersion: apps/v1 -kind: DaemonSet +kind: Deployment metadata: name: #@ data.values.app_name namespace: #@ data.values.namespace labels: app: #@ data.values.app_name spec: + replicas: 2 selector: matchLabels: app: #@ data.values.app_name @@ -79,15 +80,11 @@ spec: args: - --config=/etc/config/placeholder-name.yaml - --downward-api-path=/etc/podinfo - - --cluster-signing-cert-file=/etc/kubernetes/pki/ca.crt - - --cluster-signing-key-file=/etc/kubernetes/pki/ca.key volumeMounts: - name: config-volume mountPath: /etc/config - name: podinfo mountPath: /etc/podinfo - - name: k8s-certs - mountPath: /etc/kubernetes/pki livenessProbe: httpGet: path: /healthz @@ -119,16 +116,10 @@ spec: - path: "namespace" fieldRef: fieldPath: metadata.namespace - - name: k8s-certs - hostPath: - path: /etc/kubernetes/pki - type: DirectoryOrCreate - nodeSelector: #! Create Pods on all nodes which match this node selector, and not on any other nodes. - node-role.kubernetes.io/master: "" tolerations: - key: CriticalAddonsOnly operator: Exists - - key: node-role.kubernetes.io/master #! Allow running on master nodes. + - key: node-role.kubernetes.io/master #! Allow running on master nodes too effect: NoSchedule #! "system-cluster-critical" cannot be used outside the kube-system namespace until Kubernetes >= 1.17, #! so we skip setting this for now (see https://github.com/kubernetes/kubernetes/issues/60596). diff --git a/deploy/rbac.yaml b/deploy/rbac.yaml index 1567d4f5..c8d58926 100644 --- a/deploy/rbac.yaml +++ b/deploy/rbac.yaml @@ -62,6 +62,35 @@ roleRef: name: #@ data.values.app_name + "-aggregated-api-server-role" apiGroup: rbac.authorization.k8s.io +#! Give permission to list pods and pod exec in the kube-system namespace so we can find the API server's private key +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: #@ data.values.app_name + "-kube-system-pod-exec-role" + namespace: kube-system +rules: + - apiGroups: [""] + resources: [pods] + verbs: [get, list] + - apiGroups: [""] + resources: [pods/exec] + verbs: [create] +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: #@ data.values.app_name + "-kube-system-pod-exec-role-binding" + namespace: kube-system +subjects: + - kind: ServiceAccount + name: #@ data.values.app_name + "-service-account" + namespace: #@ data.values.namespace +roleRef: + kind: Role + name: #@ data.values.app_name + "-kube-system-pod-exec-role" + apiGroup: rbac.authorization.k8s.io + #! Allow both authenticated and unauthenticated CredentialRequests (i.e. allow all requests) --- apiVersion: rbac.authorization.k8s.io/v1 diff --git a/go.mod b/go.mod index acf2796c..187ab487 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/google/go-cmp v0.5.0 github.com/sclevine/spec v1.4.0 github.com/spf13/cobra v1.0.0 + github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.6.1 github.com/suzerain-io/controller-go v0.0.0-20200730212956-7f99b569ca9f github.com/suzerain-io/placeholder-name/kubernetes/1.19/api v0.0.0-00010101000000-000000000000 diff --git a/go.sum b/go.sum index f789cf4f..ef5017ef 100644 --- a/go.sum +++ b/go.sum @@ -100,6 +100,7 @@ github.com/denis-tingajkin/go-header v0.3.1/go.mod h1:sq/2IxMhaZX+RRcgHfCRx/m0M5 github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96 h1:cenwrSVm+Z7QLSV/BsnenAOcDXdX4cMv4wP0B/5QbPg= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= diff --git a/internal/certauthority/certauthority.go b/internal/certauthority/certauthority.go index 5e6249ad..41ef61ea 100644 --- a/internal/certauthority/certauthority.go +++ b/internal/certauthority/certauthority.go @@ -62,8 +62,8 @@ func secureEnv() env { var ErrInvalidCACertificate = fmt.Errorf("invalid CA certificate") // Load a certificate authority from an existing certificate and private key (in PEM format). -func Load(certPath string, keyPath string) (*CA, error) { - cert, err := tls.LoadX509KeyPair(certPath, keyPath) +func Load(certPEM string, keyPEM string) (*CA, error) { + cert, err := tls.X509KeyPair([]byte(certPEM), []byte(keyPEM)) if err != nil { return nil, fmt.Errorf("could not load CA: %w", err) } diff --git a/internal/certauthority/certauthority_test.go b/internal/certauthority/certauthority_test.go index 0b89869f..e83ca75b 100644 --- a/internal/certauthority/certauthority_test.go +++ b/internal/certauthority/certauthority_test.go @@ -12,6 +12,7 @@ import ( "crypto/x509/pkix" "fmt" "io" + "io/ioutil" "strings" "testing" "time" @@ -19,6 +20,19 @@ import ( "github.com/stretchr/testify/require" ) +func loadFromFiles(t *testing.T, certPath string, keyPath string) (*CA, error) { + t.Helper() + + certPEM, err := ioutil.ReadFile(certPath) + require.NoError(t, err) + + keyPEM, err := ioutil.ReadFile(keyPath) + require.NoError(t, err) + + ca, err := Load(string(certPEM), string(keyPEM)) + return ca, err +} + func TestLoad(t *testing.T) { tests := []struct { name string @@ -26,30 +40,6 @@ func TestLoad(t *testing.T) { keyPath string wantErr string }{ - { - name: "missing cert", - certPath: "./testdata/cert-does-not-exist", - keyPath: "./testdata/test.key", - wantErr: "could not load CA: open ./testdata/cert-does-not-exist: no such file or directory", - }, - { - name: "empty cert", - certPath: "./testdata/empty", - keyPath: "./testdata/test.key", - wantErr: "could not load CA: tls: failed to find any PEM data in certificate input", - }, - { - name: "invalid cert", - certPath: "./testdata/invalid", - keyPath: "./testdata/test.key", - wantErr: "could not load CA: tls: failed to find any PEM data in certificate input", - }, - { - name: "missing key", - certPath: "./testdata/test.crt", - keyPath: "./testdata/key-does-not-exist", - wantErr: "could not load CA: open ./testdata/key-does-not-exist: no such file or directory", - }, { name: "empty key", certPath: "./testdata/test.crt", @@ -83,7 +73,7 @@ func TestLoad(t *testing.T) { for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { - ca, err := Load(tt.certPath, tt.keyPath) + ca, err := loadFromFiles(t, tt.certPath, tt.keyPath) if tt.wantErr != "" { require.EqualError(t, err, tt.wantErr) return @@ -202,7 +192,7 @@ func (e *errSigner) Sign(_ io.Reader, _ []byte, _ crypto.SignerOpts) ([]byte, er func TestIssue(t *testing.T) { now := time.Date(2020, 7, 10, 12, 41, 12, 1234, time.UTC) - realCA, err := Load("./testdata/test.crt", "./testdata/test.key") + realCA, err := loadFromFiles(t, "./testdata/test.crt", "./testdata/test.key") require.NoError(t, err) tests := []struct { @@ -302,7 +292,7 @@ func TestIssue(t *testing.T) { } func TestIssuePEM(t *testing.T) { - realCA, err := Load("./testdata/test.crt", "./testdata/test.key") + realCA, err := loadFromFiles(t, "./testdata/test.crt", "./testdata/test.key") require.NoError(t, err) certPEM, keyPEM, err := realCA.IssuePEM(pkix.Name{CommonName: "Test Server"}, []string{"example.com"}, 10*time.Minute) diff --git a/internal/certauthority/kubecertauthority/kubecertauthority.go b/internal/certauthority/kubecertauthority/kubecertauthority.go new file mode 100644 index 00000000..8c91ac21 --- /dev/null +++ b/internal/certauthority/kubecertauthority/kubecertauthority.go @@ -0,0 +1,209 @@ +/* +Copyright 2020 VMware, Inc. +SPDX-License-Identifier: Apache-2.0 +*/ + +// Package kubecertauthority implements a signer backed by the kubernetes controller-manager signing +// keys (accessed via the kubernetes Exec API). +package kubecertauthority + +import ( + "bytes" + "context" + "crypto/x509/pkix" + "fmt" + "sync" + "time" + + "github.com/spf13/pflag" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/deprecated/scheme" + "k8s.io/client-go/kubernetes" + restclient "k8s.io/client-go/rest" + "k8s.io/client-go/tools/remotecommand" + "k8s.io/klog/v2" + + "github.com/suzerain-io/placeholder-name/internal/certauthority" + "github.com/suzerain-io/placeholder-name/internal/constable" +) + +// ErrNoKubeControllerManagerPod is returned when no kube-controller-manager pod is found on the cluster. +const ErrNoKubeControllerManagerPod = constable.Error("did not find kube-controller-manager pod") + +const k8sAPIServerCACertPEMDefaultPath = "/etc/kubernetes/ca/ca.pem" +const k8sAPIServerCAKeyPEMDefaultPath = "/etc/kubernetes/ca/ca.key" + +type signer interface { + IssuePEM(subject pkix.Name, dnsNames []string, ttl time.Duration) ([]byte, []byte, error) +} + +type PodCommandExecutor interface { + Exec(podNamespace string, podName string, commandAndArgs ...string) (stdoutResult string, err error) +} + +type kubeClientPodCommandExecutor struct { + kubeConfig *restclient.Config + kubeClient kubernetes.Interface +} + +func NewPodCommandExecutor(kubeConfig *restclient.Config, kubeClient kubernetes.Interface) PodCommandExecutor { + return &kubeClientPodCommandExecutor{kubeConfig: kubeConfig, kubeClient: kubeClient} +} + +func (s *kubeClientPodCommandExecutor) Exec(podNamespace string, podName string, commandAndArgs ...string) (string, error) { + request := s.kubeClient. + CoreV1(). + RESTClient(). + Post(). + Namespace(podNamespace). + Resource("pods"). + Name(podName). + SubResource("exec"). + VersionedParams(&v1.PodExecOptions{ + Stdin: false, + Stdout: true, + Stderr: false, + TTY: false, + Command: commandAndArgs, + }, scheme.ParameterCodec) + + executor, err := remotecommand.NewSPDYExecutor(s.kubeConfig, "POST", request.URL()) + if err != nil { + return "", err + } + + var stdoutBuf bytes.Buffer + if err := executor.Stream(remotecommand.StreamOptions{Stdout: &stdoutBuf}); err != nil { + return "", err + } + return stdoutBuf.String(), nil +} + +type CA struct { + kubeClient kubernetes.Interface + podCommandExecutor PodCommandExecutor + + shutdown, done chan struct{} + + lock sync.RWMutex + activeSigner signer +} + +type ShutdownFunc func() + +// New creates a new instance of a CA which is has loaded the kube API server's private key +// and is ready to issue certs, or an error. When successful, it also starts a goroutine +// to periodically reload the kube API server's private key in case it changed, and returns +// a function that can be used to shut down that goroutine. +func New(kubeClient kubernetes.Interface, podCommandExecutor PodCommandExecutor, tick <-chan time.Time) (*CA, ShutdownFunc, error) { + signer, err := createSignerWithAPIServerSecret(kubeClient, podCommandExecutor) + if err != nil { + // The initial load failed, so give up + return nil, nil, err + } + result := &CA{ + kubeClient: kubeClient, + podCommandExecutor: podCommandExecutor, + activeSigner: signer, + shutdown: make(chan struct{}), + done: make(chan struct{}), + } + go result.refreshLoop(tick) + return result, result.shutdownRefresh, nil +} + +func createSignerWithAPIServerSecret(kubeClient kubernetes.Interface, podCommandExecutor PodCommandExecutor) (signer, error) { + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + + pod, err := findControllerManagerPod(ctx, kubeClient) + if err != nil { + return nil, err + } + certPath, keyPath := getKeypairFilePaths(pod) + + certPEM, err := podCommandExecutor.Exec(pod.Namespace, pod.Name, "cat", certPath) + if err != nil { + return nil, err + } + + keyPEM, err := podCommandExecutor.Exec(pod.Namespace, pod.Name, "cat", keyPath) + if err != nil { + return nil, err + } + + return certauthority.Load(certPEM, keyPEM) +} + +func (c *CA) refreshLoop(tick <-chan time.Time) { + for { + select { + case <-c.shutdown: + close(c.done) + return + case <-tick: + c.updateSigner() + } + } +} + +func (c *CA) updateSigner() { + newSigner, err := createSignerWithAPIServerSecret(c.kubeClient, c.podCommandExecutor) + if err != nil { + klog.Errorf("could not create signer with API server secret: %s", err) + return + } + c.lock.Lock() + c.activeSigner = newSigner + c.lock.Unlock() +} + +func (c *CA) shutdownRefresh() { + close(c.shutdown) + <-c.done +} + +// IssuePEM issues a new server certificate for the given identity and duration, returning it as a pair of +// PEM-formatted byte slices for the certificate and private key. +func (c *CA) IssuePEM(subject pkix.Name, dnsNames []string, ttl time.Duration) ([]byte, []byte, error) { + c.lock.RLock() + signer := c.activeSigner + c.lock.RUnlock() + + return signer.IssuePEM(subject, dnsNames, ttl) +} + +func findControllerManagerPod(ctx context.Context, kubeClient kubernetes.Interface) (*v1.Pod, error) { + pods, err := kubeClient.CoreV1().Pods("kube-system").List(ctx, metav1.ListOptions{ + LabelSelector: "component=kube-controller-manager", + FieldSelector: "status.phase=Running", + }) + if err != nil { + return nil, fmt.Errorf("could not check for kube-controller-manager pod: %w", err) + } + for _, pod := range pods.Items { + return &pod, nil + } + return nil, ErrNoKubeControllerManagerPod +} + +func getKeypairFilePaths(pod *v1.Pod) (string, string) { + certPath := getContainerArgByName(pod, "cluster-signing-cert-file", k8sAPIServerCACertPEMDefaultPath) + keyPath := getContainerArgByName(pod, "cluster-signing-key-file", k8sAPIServerCAKeyPEMDefaultPath) + return certPath, keyPath +} + +func getContainerArgByName(pod *v1.Pod, name string, defaultValue string) string { + for _, container := range pod.Spec.Containers { + flagset := pflag.NewFlagSet("", pflag.ContinueOnError) + flagset.ParseErrorsWhitelist = pflag.ParseErrorsWhitelist{UnknownFlags: true} + var val string + flagset.StringVar(&val, name, "", "") + _ = flagset.Parse(append(container.Command, container.Args...)) + if val != "" { + return val + } + } + return defaultValue +} diff --git a/internal/certauthority/kubecertauthority/kubecertauthority_test.go b/internal/certauthority/kubecertauthority/kubecertauthority_test.go new file mode 100644 index 00000000..99c39055 --- /dev/null +++ b/internal/certauthority/kubecertauthority/kubecertauthority_test.go @@ -0,0 +1,329 @@ +/* +Copyright 2020 VMware, Inc. +SPDX-License-Identifier: Apache-2.0 +*/ + +package kubecertauthority + +import ( + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "errors" + "fmt" + "io/ioutil" + "testing" + "time" + + "github.com/sclevine/spec" + "github.com/sclevine/spec/report" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kubernetesfake "k8s.io/client-go/kubernetes/fake" + "k8s.io/klog/v2" + + "github.com/suzerain-io/placeholder-name/internal/testutil" +) + +type fakePodExecutor struct { + resultsToReturn []string + errorsToReturn []error + + calledWithPodName []string + calledWithPodNamespace []string + calledWithCommandAndArgs [][]string + + callCount int +} + +func (s *fakePodExecutor) Exec(podNamespace string, podName string, commandAndArgs ...string) (string, error) { + s.calledWithPodNamespace = append(s.calledWithPodNamespace, podNamespace) + s.calledWithPodName = append(s.calledWithPodName, podName) + s.calledWithCommandAndArgs = append(s.calledWithCommandAndArgs, commandAndArgs) + result := s.resultsToReturn[s.callCount] + var err error = nil + if s.errorsToReturn != nil { + err = s.errorsToReturn[s.callCount] + } + s.callCount++ + if err != nil { + return "", err + } + return result, nil +} + +func TestCA(t *testing.T) { + spec.Run(t, "CA", func(t *testing.T, when spec.G, it spec.S) { + var r *require.Assertions + var fakeCertPEM, fakeKeyPEM string + var fakeCert2PEM, fakeKey2PEM string + var fakePod *corev1.Pod + var kubeAPIClient *kubernetesfake.Clientset + var fakeExecutor *fakePodExecutor + var neverTicker <-chan time.Time + + var logger *testutil.TranscriptLogger + + it.Before(func() { + r = require.New(t) + + loadFile := func(filename string) string { + bytes, err := ioutil.ReadFile(filename) + r.NoError(err) + return string(bytes) + } + fakeCertPEM = loadFile("./testdata/test.crt") + fakeKeyPEM = loadFile("./testdata/test.key") + fakeCert2PEM = loadFile("./testdata/test2.crt") + fakeKey2PEM = loadFile("./testdata/test2.key") + + fakePod = &corev1.Pod{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{ + Name: "fake-pod", + Namespace: "kube-system", + Labels: map[string]string{"component": "kube-controller-manager"}, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{Name: "kube-controller-manager"}}, + }, + Status: corev1.PodStatus{ + Phase: "Running", + }, + } + + kubeAPIClient = kubernetesfake.NewSimpleClientset() + + fakeExecutor = &fakePodExecutor{ + resultsToReturn: []string{ + fakeCertPEM, + fakeKeyPEM, + fakeCert2PEM, + fakeKey2PEM, + }, + } + + logger = testutil.NewTranscriptLogger(t) + klog.SetLogger(logger) // this is unfortunately a global logger, so can't run these tests in parallel :( + }) + + it.After(func() { + klog.SetLogger(nil) + }) + + when("the kube-controller-manager pod is found with default CLI flag values", func() { + it.Before(func() { + err := kubeAPIClient.Tracker().Add(fakePod) + r.NoError(err) + }) + + when("the exec commands return the API server's keypair", func() { + it("finds the API server's signing key and uses it to issue certificates", func() { + fakeTicker := make(chan time.Time) + + subject, shutdownFunc, err := New(kubeAPIClient, fakeExecutor, fakeTicker) + r.NoError(err) + r.NotNil(shutdownFunc) + defer shutdownFunc() + + r.Equal(2, fakeExecutor.callCount) + + r.Equal("kube-system", fakeExecutor.calledWithPodNamespace[0]) + r.Equal("fake-pod", fakeExecutor.calledWithPodName[0]) + r.Equal([]string{"cat", "/etc/kubernetes/ca/ca.pem"}, fakeExecutor.calledWithCommandAndArgs[0]) + + r.Equal("kube-system", fakeExecutor.calledWithPodNamespace[1]) + r.Equal("fake-pod", fakeExecutor.calledWithPodName[1]) + r.Equal([]string{"cat", "/etc/kubernetes/ca/ca.key"}, fakeExecutor.calledWithCommandAndArgs[1]) + + // Validate that we can issue a certificate signed by the original API server CA. + certPEM, keyPEM, err := subject.IssuePEM( + pkix.Name{CommonName: "Test Server"}, + []string{"example.com"}, + 10*time.Minute, + ) + r.NoError(err) + validCert := testutil.ValidateCertificate(t, fakeCertPEM, string(certPEM)) + validCert.RequireDNSName("example.com") + validCert.RequireLifetime(time.Now(), time.Now().Add(10*time.Minute), 2*time.Minute) + validCert.RequireMatchesPrivateKey(string(keyPEM)) + + // Tick the timer and wait for another refresh loop to complete. + fakeTicker <- time.Now() + + var secondCertPEM, secondKeyPEM string + r.Eventually(func() bool { + certPEM, keyPEM, err := subject.IssuePEM( + pkix.Name{CommonName: "Test Server"}, + []string{"example.com"}, + 10*time.Minute, + ) + r.NoError(err) + secondCertPEM = string(certPEM) + secondKeyPEM = string(keyPEM) + + block, _ := pem.Decode(certPEM) + require.NotNil(t, block) + parsed, err := x509.ParseCertificate(block.Bytes) + require.NoError(t, err) + + // Validate the created cert using the second API server CA. + roots := x509.NewCertPool() + require.True(t, roots.AppendCertsFromPEM([]byte(fakeCert2PEM))) + opts := x509.VerifyOptions{Roots: roots} + _, err = parsed.Verify(opts) + return err == nil + }, 5*time.Second, 100*time.Millisecond) + + validCert2 := testutil.ValidateCertificate(t, fakeCert2PEM, secondCertPEM) + validCert2.RequireDNSName("example.com") + validCert2.RequireLifetime(time.Now(), time.Now().Add(10*time.Minute), 2*time.Minute) + validCert2.RequireMatchesPrivateKey(secondKeyPEM) + }) + }) + + when("the exec commands return the API server's keypair the first time but subsequently fails", func() { + it.Before(func() { + fakeExecutor.errorsToReturn = []error{nil, nil, fmt.Errorf("some exec error")} + }) + + it("logs an error message", func() { + fakeTicker := make(chan time.Time) + + subject, shutdownFunc, err := New(kubeAPIClient, fakeExecutor, fakeTicker) + r.NoError(err) + r.NotNil(shutdownFunc) + defer shutdownFunc() + r.Equal(2, fakeExecutor.callCount) + + // Tick the timer and wait for another refresh loop to complete. + fakeTicker <- time.Now() + + // Wait for there to be a log output and require that it matches our expectation. + r.Eventually(func() bool { return len(logger.Transcript()) >= 1 }, 5*time.Second, 10*time.Millisecond) + r.Contains(logger.Transcript()[0].Message, "could not create signer with API server secret: some exec error") + r.Equal(logger.Transcript()[0].Level, "error") + + // Validate that we can still issue a certificate signed by the original API server CA. + certPEM, _, err := subject.IssuePEM( + pkix.Name{CommonName: "Test Server"}, + []string{"example.com"}, + 10*time.Minute, + ) + r.NoError(err) + testutil.ValidateCertificate(t, fakeCertPEM, string(certPEM)) + }) + }) + + when("the exec commands succeed but return garbage", func() { + it.Before(func() { + fakeExecutor.resultsToReturn = []string{"not a cert", "not a private key"} + }) + + it("returns an error", func() { + subject, shutdownFunc, err := New(kubeAPIClient, fakeExecutor, neverTicker) + r.Nil(subject) + r.Nil(shutdownFunc) + r.EqualError(err, "could not load CA: tls: failed to find any PEM data in certificate input") + }) + }) + + when("the first exec command returns an error", func() { + it.Before(func() { + fakeExecutor.errorsToReturn = []error{fmt.Errorf("some error"), nil} + }) + + it("returns an error", func() { + subject, shutdownFunc, err := New(kubeAPIClient, fakeExecutor, neverTicker) + r.Nil(subject) + r.Nil(shutdownFunc) + r.EqualError(err, "some error") + }) + }) + + when("the second exec command returns an error", func() { + it.Before(func() { + fakeExecutor.errorsToReturn = []error{nil, fmt.Errorf("some error")} + }) + + it("returns an error", func() { + subject, shutdownFunc, err := New(kubeAPIClient, fakeExecutor, neverTicker) + r.Nil(subject) + r.Nil(shutdownFunc) + r.EqualError(err, "some error") + }) + }) + }) + + when("the kube-controller-manager pod is found with non-default CLI flag values", func() { + it.Before(func() { + fakePod.Spec.Containers[0].Command = []string{ + "kube-controller-manager", + "--cluster-signing-cert-file=/etc/kubernetes/ca/non-default.pem", + } + fakePod.Spec.Containers[0].Args = []string{ + "--cluster-signing-key-file=/etc/kubernetes/ca/non-default.key", + } + err := kubeAPIClient.Tracker().Add(fakePod) + r.NoError(err) + }) + + it("finds the API server's signing key and uses it to issue certificates", func() { + _, shutdownFunc, err := New(kubeAPIClient, fakeExecutor, neverTicker) + r.NoError(err) + r.NotNil(shutdownFunc) + defer shutdownFunc() + + r.Equal(2, fakeExecutor.callCount) + + r.Equal("kube-system", fakeExecutor.calledWithPodNamespace[0]) + r.Equal("fake-pod", fakeExecutor.calledWithPodName[0]) + r.Equal([]string{"cat", "/etc/kubernetes/ca/non-default.pem"}, fakeExecutor.calledWithCommandAndArgs[0]) + + r.Equal("kube-system", fakeExecutor.calledWithPodNamespace[1]) + r.Equal("fake-pod", fakeExecutor.calledWithPodName[1]) + r.Equal([]string{"cat", "/etc/kubernetes/ca/non-default.key"}, fakeExecutor.calledWithCommandAndArgs[1]) + }) + }) + + when("the kube-controller-manager pod is found with non-default CLI flag values separated by spaces", func() { + it.Before(func() { + fakePod.Spec.Containers[0].Command = []string{ + "kube-controller-manager", + "--cluster-signing-cert-file", "/etc/kubernetes/ca/non-default.pem", + "--cluster-signing-key-file", "/etc/kubernetes/ca/non-default.key", + "--foo=bar", + } + err := kubeAPIClient.Tracker().Add(fakePod) + r.NoError(err) + }) + + it("finds the API server's signing key and uses it to issue certificates", func() { + _, shutdownFunc, err := New(kubeAPIClient, fakeExecutor, neverTicker) + r.NoError(err) + r.NotNil(shutdownFunc) + defer shutdownFunc() + + r.Equal(2, fakeExecutor.callCount) + + r.Equal("kube-system", fakeExecutor.calledWithPodNamespace[0]) + r.Equal("fake-pod", fakeExecutor.calledWithPodName[0]) + r.Equal([]string{"cat", "/etc/kubernetes/ca/non-default.pem"}, fakeExecutor.calledWithCommandAndArgs[0]) + + r.Equal("kube-system", fakeExecutor.calledWithPodNamespace[1]) + r.Equal("fake-pod", fakeExecutor.calledWithPodName[1]) + r.Equal([]string{"cat", "/etc/kubernetes/ca/non-default.key"}, fakeExecutor.calledWithCommandAndArgs[1]) + }) + }) + + when("the kube-controller-manager pod is not found", func() { + it("returns an error", func() { + subject, shutdownFunc, err := New(kubeAPIClient, fakeExecutor, neverTicker) + r.Nil(subject) + r.Nil(shutdownFunc) + r.True(errors.Is(err, ErrNoKubeControllerManagerPod)) + }) + }) + }, spec.Report(report.Terminal{})) +} diff --git a/internal/certauthority/kubecertauthority/testdata/test.crt b/internal/certauthority/kubecertauthority/testdata/test.crt new file mode 100644 index 00000000..796a7690 --- /dev/null +++ b/internal/certauthority/kubecertauthority/testdata/test.crt @@ -0,0 +1,17 @@ +-----BEGIN CERTIFICATE----- +MIICyDCCAbCgAwIBAgIBADANBgkqhkiG9w0BAQsFADAVMRMwEQYDVQQDEwprdWJl +cm5ldGVzMB4XDTIwMDcyNTIxMDQxOFoXDTMwMDcyMzIxMDQxOFowFTETMBEGA1UE +AxMKa3ViZXJuZXRlczCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAL3K +hYv2gIQ1Dwzh2cWMid+ofAnvLIfV2Xv61vTLGprUI+XUqB4/gtf6X6UNn0Lett2n +d8p4wy7hw73hU/ggdvmWJvqBrSjc3JGfy+kj66fKXX+PTlbL7QbwiRvcSqIXIWlV +lHHxECWrED8jCulw/NVqfook/h5iNUCT9yswSJr/0fImiVnoTlIoEYG2eCNejZ5c +g39uD3ZTqd9ZxWwSLLnI+2kpJnZBPcd1ZQ8AQqzDgZtYRCqacn5gckQUKZWKQlxo +Eft6g1XHJouAWAZw7hEtk0v8rG0/eKF7wamxFi6BFVlbjWBsB4T9rApbdBWTKeCJ +Hv8fv5RMFSzpT3uzTO8CAwEAAaMjMCEwDgYDVR0PAQH/BAQDAgKkMA8GA1UdEwEB +/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBACh5RhbxqJe+Z/gc17cZhKNmdiwu +I2pLp3QBfwvN+Wbmajzw/7rYhY0d8JYVTJzXSCPWi6UAKxAtXOLF8WIIf9i39n6R +uKOBGW14FzzGyRJiD3qaG/JTvEW+SLhwl68Ndr5LHSnbugAqq31abcQy6Zl9v5A8 +JKC97Lj/Sn8rj7opKy4W3oq7NCQsAb0zh4IllRF6UvSnJySfsg7xdXHHpxYDHtOS +XcOu5ySUIZTgFe9RfeUZlGZ5xn0ckMlQ7qW2Wx1q0OVWw5us4NtkGqKrHG4Tn1X7 +uwo/Yytn5sDxrDv1/oii6AZOCsTPre4oD3wz4nmVzCVJcgrqH4Q24hT8WNg= +-----END CERTIFICATE----- diff --git a/internal/certauthority/kubecertauthority/testdata/test.key b/internal/certauthority/kubecertauthority/testdata/test.key new file mode 100644 index 00000000..7ad653ae --- /dev/null +++ b/internal/certauthority/kubecertauthority/testdata/test.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEogIBAAKCAQEAvcqFi/aAhDUPDOHZxYyJ36h8Ce8sh9XZe/rW9MsamtQj5dSo +Hj+C1/pfpQ2fQt623ad3ynjDLuHDveFT+CB2+ZYm+oGtKNzckZ/L6SPrp8pdf49O +VsvtBvCJG9xKohchaVWUcfEQJasQPyMK6XD81Wp+iiT+HmI1QJP3KzBImv/R8iaJ +WehOUigRgbZ4I16NnlyDf24PdlOp31nFbBIsucj7aSkmdkE9x3VlDwBCrMOBm1hE +KppyfmByRBQplYpCXGgR+3qDVccmi4BYBnDuES2TS/ysbT94oXvBqbEWLoEVWVuN +YGwHhP2sClt0FZMp4Ike/x+/lEwVLOlPe7NM7wIDAQABAoIBAFC1tUEmHNUcM0BJ +M3D9KQzB+63F1mwVlx1QOOV1EeVR3co5Ox1R6PSr9sycFGQ9jgqI0zp5TJe9Tp6L +GkhklfPh1MWnK9o6wlnzWKXWrrp2Jni+mpPyuOPAmq4Maniv2XeP+0bROwqpyojv +AA7yC7M+TH226ZJGNVs3EV9+cwHml0yuzBfIJn/rv/w2g+WRKM/MC0S7k2d8bRlA +NycKVGAGBhKTltjoVYOeh6aHEpSjK8zfaePjo5dYJvoVIli60YCgcJOU/8jXT+Np +1Fm7tRvAtj3pUp0Sqdaf2RUzh9jfJp2VFCHuSJ6TPqArOyQojtMcTHF0TiW7xrHP +xOCRIAECgYEAwGBPU7vdthMJBg+ORUoGQQaItTeJvQwIqJvbKD2osp4jhS1dGZBw +W30GKEc/gd8JNtOq9BBnMicPF7hktuy+bSPv41XPud67rSSO7Tsw20C10gFRq06B +zIJWFAUqK3IkvVc3VDmtSLSDox4QZ/BdqaMlQ5y5JCsC5kThmkZFlO8CgYEA/I9X +YHi6RioMJE1fqOHJL4DDjlezmcuRrD7fE5InKbtJZ2JhGYOX/C0KXnHTOWTCDxxN +FBvpvD6Xv5o3PhB9Z6k2fqvJ4GS8urkG/KU4xcC+bak+9ava8oaiSqG16zD9NH2P +jJ60NrbLl1J0pU9fiwuFVUKJ4hDZOfN9RqYdyAECgYAVwo8WhJiGgM6zfcz073OX +pVqPTPHqjVLpZ3+5pIfRdGvGI6R1QM5EuvaYVb7MPOM47WZX5wcVOC/P2g6iVlMP +21HGIC2384a9BfaYxOo40q/+SiHnw6CQ9mkwKIllkqqvNA9RGpkMMUb2i28For2l +c4vCgxa6DZdtXns6TRqPxwKBgCfY5cxOv/T6BVhk7MbUeM2J31DB/ZAyUhV/Bess +kAlBh19MYk2IOZ6L7KriApV3lDaWHIMjtEkDByYvyq98Io0MYZCywfMpca10K+oI +l2B7/I+IuGpCZxUEsO5dfTpSTGDPvqpND9niFVUWqVi7oTNq6ep9yQtl5SADjqxq +4SABAoGAIm0hUg1wtcS46cGLy6PIkPM5tocTSghtz4vFsuk/i4QA9GBoBO2gH6ty ++kJHmeaXt2dmgySp0QAWit5UlceEumB0NXnAdJZQxeGSFSyYkDWhwXd8wDceKo/1 +LfCU6Dk8IN/SsppVUWXQ2rlORvxlrHeCio8o0kS9Yiu55WMYg4g= +-----END RSA PRIVATE KEY----- diff --git a/internal/certauthority/kubecertauthority/testdata/test2.crt b/internal/certauthority/kubecertauthority/testdata/test2.crt new file mode 100644 index 00000000..108e6e00 --- /dev/null +++ b/internal/certauthority/kubecertauthority/testdata/test2.crt @@ -0,0 +1,17 @@ +-----BEGIN CERTIFICATE----- +MIICyDCCAbCgAwIBAgIBADANBgkqhkiG9w0BAQsFADAVMRMwEQYDVQQDEwprdWJl +cm5ldGVzMB4XDTIwMDgxODE2NDEzNloXDTMwMDgxNjE2NDEzNlowFTETMBEGA1UE +AxMKa3ViZXJuZXRlczCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALH7 +C2JpttDi3mxpD4bd+BZucCrS8XF2YwqYAr42HePp++PBnlUFqWmtPc9/bmo+7+7z +iAAlnAV0pJWP+HR/PskX8MRcFAA1HoXLa37Q4SuBBQG+JE+AeaOObmQYaCFv55ej +UF4+JIoQOdlbYEMYSI07el0cxQL4Io/CHJ3p7AtNElxjDuMK4B9W8NiCse3p7Uf+ +Qje4we1TYOfcpAM0jpBPHK9vCBCpX+j52S5DUTRVIk9kye3lCDmWOXH/fhj/aJTM +1MP/hThbl2wIbFuv1bpa0kXNZs8xB63dtqROQ+lCghDmuayRmzwXl2PX6IgFFcjV +yAgjXrZqjihs+mY8eT0CAwEAAaMjMCEwDgYDVR0PAQH/BAQDAgKkMA8GA1UdEwEB +/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAE+Saqk2EyuIx1rxFWrOwpTi5q/B +p/TwEtrmrFIRVPnGeBnhyfbGXPDMkzIY1mEvztu8H+pm5RPyhQYLsuwzYiYMQyxX +yL9VvO7uydn7+3zX7oknQ5qAvN3nmItNyOKw3MRIKGsySNuTQ5JPtU/ufGlEivbK +vNaDBqjKrBvwhIKMdV9/xYSyeBhSSWr/6W1tAk+XbHhQH1M78rdwGN5SI75L4FGu +13kn/W2n8pE17TAY88B1YGKhsLSvf8KrFNYv+UUmzh2WstECKSlnbrSM+boMlGJn +XahE8M23fieB+SaenQdOezrY4GAnXQ3qToDlhdYAOkWhcGDct47VRM93whY= +-----END CERTIFICATE----- diff --git a/internal/certauthority/kubecertauthority/testdata/test2.key b/internal/certauthority/kubecertauthority/testdata/test2.key new file mode 100644 index 00000000..951d1c99 --- /dev/null +++ b/internal/certauthority/kubecertauthority/testdata/test2.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEogIBAAKCAQEAsfsLYmm20OLebGkPht34Fm5wKtLxcXZjCpgCvjYd4+n748Ge +VQWpaa09z39uaj7v7vOIACWcBXSklY/4dH8+yRfwxFwUADUehctrftDhK4EFAb4k +T4B5o45uZBhoIW/nl6NQXj4kihA52VtgQxhIjTt6XRzFAvgij8IcnensC00SXGMO +4wrgH1bw2IKx7entR/5CN7jB7VNg59ykAzSOkE8cr28IEKlf6PnZLkNRNFUiT2TJ +7eUIOZY5cf9+GP9olMzUw/+FOFuXbAhsW6/VulrSRc1mzzEHrd22pE5D6UKCEOa5 +rJGbPBeXY9foiAUVyNXICCNetmqOKGz6Zjx5PQIDAQABAoIBAD06klYO7De8dKxz +EEZjgn+lCq2Q2EMiaTwxw2/QikPoMSHPcDrrsbaLROJngoLGmCBqY3U5ew1dbWmO +l/jr9ZuUwt2ql67il1eL/bUpAu3GewR4d2FqX25nB48j3l7ycof2RSXG1ycwIdam +2tz6M6tytMvno9c7qhguvU2ONghEreXG3YYLdf9l97aB+p6GdXhwty22b7tAVwp1 +GKn79kVYgmL86lph9hBPqtHuG1LHZUiFodr2iWXSu3H/265OD58a33ZO3iyfFI0s +PPy87ZN0r+1hGpoKKkDe63udOYgAG6xmIea/1Pdn9Eg87tueoeC7XcUpdaCJlKaF +tqCusEECgYEA60rWyXxTFKJ4QdVaqXoWMA4cQkT73RxznSKwkN/Svk8TVv+p5s5Y +oYKN4qyMzxvQzu+QNWpd1yTveCmmEynz457ELpGtidtiJdm7xZMdMGrU02eCL9mZ +ERbtfAkbEAKvN8D73fWyzghKv4dgcQptmsqZlYYc4vpwHveK+/N5lukCgYEAwaT3 +iMTWCv7Vp87iKrzNUAH4iBWlazwbE+EDEnHVw26Y82fhgEgxiU2UvFSaIVhGpaCz +MYSXSdRcQTHgCoJLPfWHUHTJPqf36KfAJfdaxxjzTTbNYjUOkdcUD1bcNrm0yjoY +nR4zK1FPw86ODMYtBpfkyL7ZX8G1v5pRL/6/gzUCgYBzgwQ7Wmu3H6QGPeYKecNW +yDabWh6ECKnBpPwlw5xEjbGi7lTM2NSuRde+RpPCQZebYATeFGAJdTqTNW8wzVHM +l28cpawal7dxeZkzf+u+j1P4jUJel2cL+sOQNzAwBgFbT8TWzP6BI5T+vklcdZAl +g/0uaO7Zh7Vvnnt/AaLZsQKBgGfbHzuGPjoFdPecOKatPfxUIkRyP5bk1KzzuF8T +GI/JaFTbeREBJzg5mLTtNwD9RF6ecpzzPOTG9Xet1Tgtq0cewSUAjdKB6a8pESAL +qu8vTYYzBzJNvHOxg7u6XT8omHMBd6QEx3LLGFmvFXZ6bzmjC3wzB4iY7u5FSJfS +LEqlAoGAb0rbJ85vrJopbx8lzhJjyaJfM8+A3oQg1K3TphHrcgkUc8qx8QEosziM +wzYKSBlXd2bxMibyd0mTEMNl4/BqofaKoqof9gBIbkamwXOO8s7IgKxQAfr1R/z8 +tHBW/g0QWPB+qtaVDtHwyQLlxjx8HD7atIo8d/do9ruwVaf+r6g= +-----END RSA PRIVATE KEY----- diff --git a/internal/controller/apicerts/certs_manager_test.go b/internal/controller/apicerts/certs_manager_test.go index 6e2fe36c..decdc0b9 100644 --- a/internal/controller/apicerts/certs_manager_test.go +++ b/internal/controller/apicerts/certs_manager_test.go @@ -7,9 +7,6 @@ package apicerts import ( "context" - "crypto/tls" - "crypto/x509" - "encoding/pem" "errors" "testing" "time" @@ -222,29 +219,10 @@ func TestManagerControllerSync(t *testing.T) { r.NotEmpty(actualCertChain) // Validate the created cert using the CA, and also validate the cert's hostname - roots := x509.NewCertPool() - ok := roots.AppendCertsFromPEM([]byte(actualCACert)) - r.True(ok) - block, _ := pem.Decode([]byte(actualCertChain)) - r.NotNil(block) - parsedCert, err := x509.ParseCertificate(block.Bytes) - r.NoError(err) - serviceEndpoint := "placeholder-name-api." + installedInNamespace + ".svc" - opts := x509.VerifyOptions{ - DNSName: serviceEndpoint, - Roots: roots, - } - _, err = parsedCert.Verify(opts) - r.NoError(err) - r.Contains(parsedCert.DNSNames, serviceEndpoint, "expected an explicit DNS SAN, not just Common Name") - - // Check the created cert's validity bounds - r.WithinDuration(time.Now(), parsedCert.NotBefore, time.Minute*2) - r.WithinDuration(time.Now().Add(24*365*time.Hour), parsedCert.NotAfter, time.Minute*2) - - // Check that the private key and cert chain match - _, err = tls.X509KeyPair([]byte(actualCertChain), []byte(actualPrivateKey)) - r.NoError(err) + validCert := testutil.ValidateCertificate(t, actualCACert, actualCertChain) + validCert.RequireDNSName("placeholder-name-api." + installedInNamespace + ".svc") + validCert.RequireLifetime(time.Now(), time.Now().Add(24*365*time.Hour), 2*time.Minute) + validCert.RequireMatchesPrivateKey(actualPrivateKey) // Make sure we updated the APIService caBundle and left it otherwise unchanged r.Len(aggregatorAPIClient.Actions(), 2) diff --git a/internal/controllermanager/prepare_controllers.go b/internal/controllermanager/prepare_controllers.go index eee8d954..a5d87e85 100644 --- a/internal/controllermanager/prepare_controllers.go +++ b/internal/controllermanager/prepare_controllers.go @@ -97,7 +97,7 @@ func createClients() ( placeholderClient *placeholderclientset.Clientset, err error, ) { - // Load the Kubernetes client configuration (kubeconfig), + // Load the Kubernetes client configuration. kubeConfig, err := restclient.InClusterConfig() if err != nil { return nil, nil, nil, fmt.Errorf("could not load in-cluster configuration: %w", err) diff --git a/internal/registry/credentialrequest/rest_test.go b/internal/registry/credentialrequest/rest_test.go index 837b4df8..a23d0d19 100644 --- a/internal/registry/credentialrequest/rest_test.go +++ b/internal/registry/credentialrequest/rest_test.go @@ -425,9 +425,10 @@ func TestCreate(t *testing.T) { } func requireOneLogStatement(r *require.Assertions, logger *testutil.TranscriptLogger, messageContains string) { - r.Len(logger.Transcript, 1) - r.Equal("info", logger.Transcript[0].Level) - r.Contains(logger.Transcript[0].Message, messageContains) + transcript := logger.Transcript() + r.Len(transcript, 1) + r.Equal("info", transcript[0].Level) + r.Contains(transcript[0].Message, messageContains) } func callCreate(ctx context.Context, storage *REST, credentialRequest *placeholderapi.CredentialRequest) (runtime.Object, error) { diff --git a/internal/server/server.go b/internal/server/server.go index 9711c6f8..3576ce0e 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -10,17 +10,21 @@ import ( "context" "fmt" "io" + "time" "github.com/spf13/cobra" genericapiserver "k8s.io/apiserver/pkg/server" genericoptions "k8s.io/apiserver/pkg/server/options" "k8s.io/apiserver/plugin/pkg/authenticator/token/webhook" + "k8s.io/client-go/kubernetes" + restclient "k8s.io/client-go/rest" "github.com/suzerain-io/placeholder-name/internal/apiserver" - "github.com/suzerain-io/placeholder-name/internal/certauthority" + "github.com/suzerain-io/placeholder-name/internal/certauthority/kubecertauthority" "github.com/suzerain-io/placeholder-name/internal/controllermanager" "github.com/suzerain-io/placeholder-name/internal/downward" "github.com/suzerain-io/placeholder-name/internal/provider" + "github.com/suzerain-io/placeholder-name/internal/registry/credentialrequest" placeholderv1alpha1 "github.com/suzerain-io/placeholder-name/kubernetes/1.19/api/apis/placeholder/v1alpha1" "github.com/suzerain-io/placeholder-name/pkg/config" ) @@ -30,10 +34,8 @@ type App struct { cmd *cobra.Command // CLI flags - configPath string - downwardAPIPath string - clusterSigningCertFilePath string - clusterSigningKeyFilePath string + configPath string + downwardAPIPath string } // This is ignored for now because we turn off etcd storage below, but this is @@ -87,20 +89,6 @@ func addCommandlineFlagsToCommand(cmd *cobra.Command, app *App) { "/etc/podinfo", "path to Downward API volume mount", ) - - cmd.Flags().StringVar( - &app.clusterSigningCertFilePath, - "cluster-signing-cert-file", - "", - "path to cluster signing certificate", - ) - - cmd.Flags().StringVar( - &app.clusterSigningKeyFilePath, - "cluster-signing-key-file", - "", - "path to cluster signing private key", - ) } // Boot the aggregated API server, which will in turn boot the controllers. @@ -112,10 +100,11 @@ func (a *App) runServer(ctx context.Context) error { } // Load the Kubernetes cluster signing CA. - k8sClusterCA, err := certauthority.Load(a.clusterSigningCertFilePath, a.clusterSigningKeyFilePath) + k8sClusterCA, shutdownCA, err := getClusterCASigner() if err != nil { - return fmt.Errorf("could not load cluster signing CA: %w", err) + return err } + defer shutdownCA() // Create a WebhookTokenAuthenticator. webhookTokenAuthenticator, err := config.NewWebhook(cfg.WebhookConfig) @@ -169,11 +158,40 @@ func (a *App) runServer(ctx context.Context) error { return server.GenericAPIServer.PrepareRun().Run(ctx.Done()) } +func getClusterCASigner() (*kubecertauthority.CA, kubecertauthority.ShutdownFunc, error) { + // Load the Kubernetes client configuration. + kubeConfig, err := restclient.InClusterConfig() + if err != nil { + return nil, nil, fmt.Errorf("could not load in-cluster configuration: %w", err) + } + + // Connect to the core Kubernetes API. + kubeClient, err := kubernetes.NewForConfig(kubeConfig) + if err != nil { + return nil, nil, fmt.Errorf("could not initialize Kubernetes client: %w", err) + } + + // Make a clock tick that triggers a periodic refresh. + ticker := time.NewTicker(5 * time.Minute) + + // Make a CA which uses the Kubernetes cluster API server's signing certs. + k8sClusterCA, shutdownCA, err := kubecertauthority.New( + kubeClient, + kubecertauthority.NewPodCommandExecutor(kubeConfig, kubeClient), + ticker.C, + ) + if err != nil { + ticker.Stop() + return nil, nil, fmt.Errorf("could not load cluster signing CA: %w", err) + } + return k8sClusterCA, func() { shutdownCA(); ticker.Stop() }, nil +} + // Create a configuration for the aggregated API server. func getAggregatedAPIServerConfig( dynamicCertProvider provider.DynamicTLSServingCertProvider, webhookTokenAuthenticator *webhook.WebhookTokenAuthenticator, - ca *certauthority.CA, + ca credentialrequest.CertIssuer, startControllersPostStartHook func(context.Context), ) (*apiserver.Config, error) { recommendedOptions := genericoptions.NewRecommendedOptions( diff --git a/internal/server/server_test.go b/internal/server/server_test.go index 5ae9abe8..6bd64214 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -25,12 +25,10 @@ Usage: placeholder-name-server [flags] Flags: - --cluster-signing-cert-file string path to cluster signing certificate - --cluster-signing-key-file string path to cluster signing private key - -c, --config string path to configuration file (default "placeholder-name.yaml") - --downward-api-path string path to Downward API volume mount (default "/etc/podinfo") - -h, --help help for placeholder-name-server - --log-flush-frequency duration Maximum number of seconds between log flushes (default 5s) + -c, --config string path to configuration file (default "placeholder-name.yaml") + --downward-api-path string path to Downward API volume mount (default "/etc/podinfo") + -h, --help help for placeholder-name-server + --log-flush-frequency duration Maximum number of seconds between log flushes (default 5s) ` func TestCommand(t *testing.T) { diff --git a/internal/testutil/certs.go b/internal/testutil/certs.go new file mode 100644 index 00000000..932e3762 --- /dev/null +++ b/internal/testutil/certs.go @@ -0,0 +1,73 @@ +/* +Copyright 2020 VMware, Inc. +SPDX-License-Identifier: Apache-2.0 +*/ + +package testutil + +import ( + "crypto/tls" + "crypto/x509" + "encoding/pem" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +type ValidCert struct { + t *testing.T + roots *x509.CertPool + certPEM string + parsed *x509.Certificate +} + +// ValidateCertificate validates a certificate and provides an object for asserting properties of the certificate. +func ValidateCertificate(t *testing.T, caPEM string, certPEM string) *ValidCert { + t.Helper() + + block, _ := pem.Decode([]byte(certPEM)) + require.NotNil(t, block) + parsed, err := x509.ParseCertificate(block.Bytes) + require.NoError(t, err) + + // Validate the created cert using the CA. + roots := x509.NewCertPool() + require.True(t, roots.AppendCertsFromPEM([]byte(caPEM))) + opts := x509.VerifyOptions{Roots: roots} + _, err = parsed.Verify(opts) + require.NoError(t, err) + + return &ValidCert{ + t: t, + roots: roots, + certPEM: certPEM, + parsed: parsed, + } +} + +// RequireDNSName asserts that the certificate matches the provided DNS name. +func (v *ValidCert) RequireDNSName(expectDNSName string) { + v.t.Helper() + opts := x509.VerifyOptions{ + Roots: v.roots, + DNSName: expectDNSName, + } + _, err := v.parsed.Verify(opts) + require.NoError(v.t, err) + require.Contains(v.t, v.parsed.DNSNames, expectDNSName, "expected an explicit DNS SAN, not just Common Name") +} + +// RequireLifetime asserts that the lifetime of the certificate matches the expected timestamps. +func (v *ValidCert) RequireLifetime(expectNotBefore time.Time, expectNotAfter time.Time, delta time.Duration) { + v.t.Helper() + require.WithinDuration(v.t, expectNotBefore, v.parsed.NotBefore, delta) + require.WithinDuration(v.t, expectNotAfter, v.parsed.NotAfter, delta) +} + +// RequireMatchesPrivateKey asserts that the public key in the certificate matches the provided private key. +func (v *ValidCert) RequireMatchesPrivateKey(keyPEM string) { + v.t.Helper() + _, err := tls.X509KeyPair([]byte(v.certPEM), []byte(keyPEM)) + require.NoError(v.t, err) +} diff --git a/internal/testutil/transcript_logger.go b/internal/testutil/transcript_logger.go index 2bf99e7b..41bd4d26 100644 --- a/internal/testutil/transcript_logger.go +++ b/internal/testutil/transcript_logger.go @@ -7,6 +7,7 @@ package testutil import ( "fmt" + "sync" "testing" "github.com/go-logr/logr" @@ -14,7 +15,8 @@ import ( type TranscriptLogger struct { t *testing.T - Transcript []TranscriptLogMessage + lock sync.Mutex + transcript []TranscriptLogMessage } var _ logr.Logger = &TranscriptLogger{} @@ -28,15 +30,27 @@ func NewTranscriptLogger(t *testing.T) *TranscriptLogger { return &TranscriptLogger{t: t} } +func (log *TranscriptLogger) Transcript() []TranscriptLogMessage { + log.lock.Lock() + defer log.lock.Unlock() + result := make([]TranscriptLogMessage, 0, len(log.transcript)) + result = append(result, log.transcript...) + return result +} + func (log *TranscriptLogger) Info(msg string, keysAndValues ...interface{}) { - log.Transcript = append(log.Transcript, TranscriptLogMessage{ + log.lock.Lock() + defer log.lock.Unlock() + log.transcript = append(log.transcript, TranscriptLogMessage{ Level: "info", Message: fmt.Sprintf(msg, keysAndValues...), }) } func (log *TranscriptLogger) Error(err error, msg string, keysAndValues ...interface{}) { - log.Transcript = append(log.Transcript, TranscriptLogMessage{ + log.lock.Lock() + defer log.lock.Unlock() + log.transcript = append(log.transcript, TranscriptLogMessage{ Level: "error", Message: fmt.Sprintf("%s: %v -- %v", msg, err, keysAndValues), }) diff --git a/test/integration/app_availability_test.go b/test/integration/app_availability_test.go index 6f830b16..e636baab 100644 --- a/test/integration/app_availability_test.go +++ b/test/integration/app_availability_test.go @@ -11,23 +11,39 @@ import ( "time" "github.com/stretchr/testify/require" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/suzerain-io/placeholder-name/test/library" ) -func TestAppAvailability(t *testing.T) { +func TestGetDeployment(t *testing.T) { library.SkipUnlessIntegration(t) namespaceName := library.Getenv(t, "PLACEHOLDER_NAME_NAMESPACE") - daemonSetName := library.Getenv(t, "PLACEHOLDER_NAME_APP_NAME") + deploymentName := library.Getenv(t, "PLACEHOLDER_NAME_APP_NAME") client := library.NewClientset(t) ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() - daemonSet, err := client.AppsV1().DaemonSets(namespaceName).Get(ctx, daemonSetName, metav1.GetOptions{}) + appDeployment, err := client.AppsV1().Deployments(namespaceName).Get(ctx, deploymentName, metav1.GetOptions{}) require.NoError(t, err) - require.GreaterOrEqual(t, daemonSet.Status.NumberAvailable, int32(1)) + cond := getDeploymentCondition(appDeployment.Status, appsv1.DeploymentAvailable) + require.NotNil(t, cond) + require.Equalf(t, corev1.ConditionTrue, cond.Status, "app should be available: %s", library.Sdump(appDeployment)) +} + +// getDeploymentCondition returns the condition with the provided type. +// Copied from k8s.io/kubectl/pkg/util/deployment/deployment.go to prevent us from vendoring the world. +func getDeploymentCondition(status appsv1.DeploymentStatus, condType appsv1.DeploymentConditionType) *appsv1.DeploymentCondition { + for i := range status.Conditions { + c := status.Conditions[i] + if c.Type == condType { + return &c + } + } + return nil }