diff --git a/deploy/deployment.yaml b/deploy/deployment.yaml index 48c9b8d8..5f752f46 100644 --- a/deploy/deployment.yaml +++ b/deploy/deployment.yaml @@ -46,13 +46,14 @@ data: #! TODO set up healthy, ready, etc. probes correctly? #! TODO set resource minimums (e.g. 512MB RAM) to make sure we get scheduled onto a reasonable node? apiVersion: apps/v1 -kind: DaemonSet +kind: Deployment metadata: name: #@ data.values.app_name namespace: #@ data.values.namespace labels: app: #@ data.values.app_name spec: + replicas: 2 selector: matchLabels: app: #@ data.values.app_name @@ -79,15 +80,11 @@ spec: args: - --config=/etc/config/placeholder-name.yaml - --downward-api-path=/etc/podinfo - - --cluster-signing-cert-file=/etc/kubernetes/pki/ca.crt - - --cluster-signing-key-file=/etc/kubernetes/pki/ca.key volumeMounts: - name: config-volume mountPath: /etc/config - name: podinfo mountPath: /etc/podinfo - - name: k8s-certs - mountPath: /etc/kubernetes/pki livenessProbe: httpGet: path: /healthz @@ -119,16 +116,10 @@ spec: - path: "namespace" fieldRef: fieldPath: metadata.namespace - - name: k8s-certs - hostPath: - path: /etc/kubernetes/pki - type: DirectoryOrCreate - nodeSelector: #! Create Pods on all nodes which match this node selector, and not on any other nodes. - node-role.kubernetes.io/master: "" tolerations: - key: CriticalAddonsOnly operator: Exists - - key: node-role.kubernetes.io/master #! Allow running on master nodes. + - key: node-role.kubernetes.io/master #! Allow running on master nodes too effect: NoSchedule #! "system-cluster-critical" cannot be used outside the kube-system namespace until Kubernetes >= 1.17, #! so we skip setting this for now (see https://github.com/kubernetes/kubernetes/issues/60596). diff --git a/deploy/rbac.yaml b/deploy/rbac.yaml index 1567d4f5..c8d58926 100644 --- a/deploy/rbac.yaml +++ b/deploy/rbac.yaml @@ -62,6 +62,35 @@ roleRef: name: #@ data.values.app_name + "-aggregated-api-server-role" apiGroup: rbac.authorization.k8s.io +#! Give permission to list pods and pod exec in the kube-system namespace so we can find the API server's private key +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: #@ data.values.app_name + "-kube-system-pod-exec-role" + namespace: kube-system +rules: + - apiGroups: [""] + resources: [pods] + verbs: [get, list] + - apiGroups: [""] + resources: [pods/exec] + verbs: [create] +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: #@ data.values.app_name + "-kube-system-pod-exec-role-binding" + namespace: kube-system +subjects: + - kind: ServiceAccount + name: #@ data.values.app_name + "-service-account" + namespace: #@ data.values.namespace +roleRef: + kind: Role + name: #@ data.values.app_name + "-kube-system-pod-exec-role" + apiGroup: rbac.authorization.k8s.io + #! Allow both authenticated and unauthenticated CredentialRequests (i.e. allow all requests) --- apiVersion: rbac.authorization.k8s.io/v1 diff --git a/go.mod b/go.mod index acf2796c..187ab487 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/google/go-cmp v0.5.0 github.com/sclevine/spec v1.4.0 github.com/spf13/cobra v1.0.0 + github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.6.1 github.com/suzerain-io/controller-go v0.0.0-20200730212956-7f99b569ca9f github.com/suzerain-io/placeholder-name/kubernetes/1.19/api v0.0.0-00010101000000-000000000000 diff --git a/go.sum b/go.sum index f789cf4f..ef5017ef 100644 --- a/go.sum +++ b/go.sum @@ -100,6 +100,7 @@ github.com/denis-tingajkin/go-header v0.3.1/go.mod h1:sq/2IxMhaZX+RRcgHfCRx/m0M5 github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96 h1:cenwrSVm+Z7QLSV/BsnenAOcDXdX4cMv4wP0B/5QbPg= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= diff --git a/internal/certauthority/certauthority.go b/internal/certauthority/certauthority.go index 5e6249ad..41ef61ea 100644 --- a/internal/certauthority/certauthority.go +++ b/internal/certauthority/certauthority.go @@ -62,8 +62,8 @@ func secureEnv() env { var ErrInvalidCACertificate = fmt.Errorf("invalid CA certificate") // Load a certificate authority from an existing certificate and private key (in PEM format). -func Load(certPath string, keyPath string) (*CA, error) { - cert, err := tls.LoadX509KeyPair(certPath, keyPath) +func Load(certPEM string, keyPEM string) (*CA, error) { + cert, err := tls.X509KeyPair([]byte(certPEM), []byte(keyPEM)) if err != nil { return nil, fmt.Errorf("could not load CA: %w", err) } diff --git a/internal/certauthority/certauthority_test.go b/internal/certauthority/certauthority_test.go index 0b89869f..e83ca75b 100644 --- a/internal/certauthority/certauthority_test.go +++ b/internal/certauthority/certauthority_test.go @@ -12,6 +12,7 @@ import ( "crypto/x509/pkix" "fmt" "io" + "io/ioutil" "strings" "testing" "time" @@ -19,6 +20,19 @@ import ( "github.com/stretchr/testify/require" ) +func loadFromFiles(t *testing.T, certPath string, keyPath string) (*CA, error) { + t.Helper() + + certPEM, err := ioutil.ReadFile(certPath) + require.NoError(t, err) + + keyPEM, err := ioutil.ReadFile(keyPath) + require.NoError(t, err) + + ca, err := Load(string(certPEM), string(keyPEM)) + return ca, err +} + func TestLoad(t *testing.T) { tests := []struct { name string @@ -26,30 +40,6 @@ func TestLoad(t *testing.T) { keyPath string wantErr string }{ - { - name: "missing cert", - certPath: "./testdata/cert-does-not-exist", - keyPath: "./testdata/test.key", - wantErr: "could not load CA: open ./testdata/cert-does-not-exist: no such file or directory", - }, - { - name: "empty cert", - certPath: "./testdata/empty", - keyPath: "./testdata/test.key", - wantErr: "could not load CA: tls: failed to find any PEM data in certificate input", - }, - { - name: "invalid cert", - certPath: "./testdata/invalid", - keyPath: "./testdata/test.key", - wantErr: "could not load CA: tls: failed to find any PEM data in certificate input", - }, - { - name: "missing key", - certPath: "./testdata/test.crt", - keyPath: "./testdata/key-does-not-exist", - wantErr: "could not load CA: open ./testdata/key-does-not-exist: no such file or directory", - }, { name: "empty key", certPath: "./testdata/test.crt", @@ -83,7 +73,7 @@ func TestLoad(t *testing.T) { for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { - ca, err := Load(tt.certPath, tt.keyPath) + ca, err := loadFromFiles(t, tt.certPath, tt.keyPath) if tt.wantErr != "" { require.EqualError(t, err, tt.wantErr) return @@ -202,7 +192,7 @@ func (e *errSigner) Sign(_ io.Reader, _ []byte, _ crypto.SignerOpts) ([]byte, er func TestIssue(t *testing.T) { now := time.Date(2020, 7, 10, 12, 41, 12, 1234, time.UTC) - realCA, err := Load("./testdata/test.crt", "./testdata/test.key") + realCA, err := loadFromFiles(t, "./testdata/test.crt", "./testdata/test.key") require.NoError(t, err) tests := []struct { @@ -302,7 +292,7 @@ func TestIssue(t *testing.T) { } func TestIssuePEM(t *testing.T) { - realCA, err := Load("./testdata/test.crt", "./testdata/test.key") + realCA, err := loadFromFiles(t, "./testdata/test.crt", "./testdata/test.key") require.NoError(t, err) certPEM, keyPEM, err := realCA.IssuePEM(pkix.Name{CommonName: "Test Server"}, []string{"example.com"}, 10*time.Minute) diff --git a/internal/certauthority/kubecertauthority/kubecertauthority.go b/internal/certauthority/kubecertauthority/kubecertauthority.go new file mode 100644 index 00000000..8c91ac21 --- /dev/null +++ b/internal/certauthority/kubecertauthority/kubecertauthority.go @@ -0,0 +1,209 @@ +/* +Copyright 2020 VMware, Inc. +SPDX-License-Identifier: Apache-2.0 +*/ + +// Package kubecertauthority implements a signer backed by the kubernetes controller-manager signing +// keys (accessed via the kubernetes Exec API). +package kubecertauthority + +import ( + "bytes" + "context" + "crypto/x509/pkix" + "fmt" + "sync" + "time" + + "github.com/spf13/pflag" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/deprecated/scheme" + "k8s.io/client-go/kubernetes" + restclient "k8s.io/client-go/rest" + "k8s.io/client-go/tools/remotecommand" + "k8s.io/klog/v2" + + "github.com/suzerain-io/placeholder-name/internal/certauthority" + "github.com/suzerain-io/placeholder-name/internal/constable" +) + +// ErrNoKubeControllerManagerPod is returned when no kube-controller-manager pod is found on the cluster. +const ErrNoKubeControllerManagerPod = constable.Error("did not find kube-controller-manager pod") + +const k8sAPIServerCACertPEMDefaultPath = "/etc/kubernetes/ca/ca.pem" +const k8sAPIServerCAKeyPEMDefaultPath = "/etc/kubernetes/ca/ca.key" + +type signer interface { + IssuePEM(subject pkix.Name, dnsNames []string, ttl time.Duration) ([]byte, []byte, error) +} + +type PodCommandExecutor interface { + Exec(podNamespace string, podName string, commandAndArgs ...string) (stdoutResult string, err error) +} + +type kubeClientPodCommandExecutor struct { + kubeConfig *restclient.Config + kubeClient kubernetes.Interface +} + +func NewPodCommandExecutor(kubeConfig *restclient.Config, kubeClient kubernetes.Interface) PodCommandExecutor { + return &kubeClientPodCommandExecutor{kubeConfig: kubeConfig, kubeClient: kubeClient} +} + +func (s *kubeClientPodCommandExecutor) Exec(podNamespace string, podName string, commandAndArgs ...string) (string, error) { + request := s.kubeClient. + CoreV1(). + RESTClient(). + Post(). + Namespace(podNamespace). + Resource("pods"). + Name(podName). + SubResource("exec"). + VersionedParams(&v1.PodExecOptions{ + Stdin: false, + Stdout: true, + Stderr: false, + TTY: false, + Command: commandAndArgs, + }, scheme.ParameterCodec) + + executor, err := remotecommand.NewSPDYExecutor(s.kubeConfig, "POST", request.URL()) + if err != nil { + return "", err + } + + var stdoutBuf bytes.Buffer + if err := executor.Stream(remotecommand.StreamOptions{Stdout: &stdoutBuf}); err != nil { + return "", err + } + return stdoutBuf.String(), nil +} + +type CA struct { + kubeClient kubernetes.Interface + podCommandExecutor PodCommandExecutor + + shutdown, done chan struct{} + + lock sync.RWMutex + activeSigner signer +} + +type ShutdownFunc func() + +// New creates a new instance of a CA which is has loaded the kube API server's private key +// and is ready to issue certs, or an error. When successful, it also starts a goroutine +// to periodically reload the kube API server's private key in case it changed, and returns +// a function that can be used to shut down that goroutine. +func New(kubeClient kubernetes.Interface, podCommandExecutor PodCommandExecutor, tick <-chan time.Time) (*CA, ShutdownFunc, error) { + signer, err := createSignerWithAPIServerSecret(kubeClient, podCommandExecutor) + if err != nil { + // The initial load failed, so give up + return nil, nil, err + } + result := &CA{ + kubeClient: kubeClient, + podCommandExecutor: podCommandExecutor, + activeSigner: signer, + shutdown: make(chan struct{}), + done: make(chan struct{}), + } + go result.refreshLoop(tick) + return result, result.shutdownRefresh, nil +} + +func createSignerWithAPIServerSecret(kubeClient kubernetes.Interface, podCommandExecutor PodCommandExecutor) (signer, error) { + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + + pod, err := findControllerManagerPod(ctx, kubeClient) + if err != nil { + return nil, err + } + certPath, keyPath := getKeypairFilePaths(pod) + + certPEM, err := podCommandExecutor.Exec(pod.Namespace, pod.Name, "cat", certPath) + if err != nil { + return nil, err + } + + keyPEM, err := podCommandExecutor.Exec(pod.Namespace, pod.Name, "cat", keyPath) + if err != nil { + return nil, err + } + + return certauthority.Load(certPEM, keyPEM) +} + +func (c *CA) refreshLoop(tick <-chan time.Time) { + for { + select { + case <-c.shutdown: + close(c.done) + return + case <-tick: + c.updateSigner() + } + } +} + +func (c *CA) updateSigner() { + newSigner, err := createSignerWithAPIServerSecret(c.kubeClient, c.podCommandExecutor) + if err != nil { + klog.Errorf("could not create signer with API server secret: %s", err) + return + } + c.lock.Lock() + c.activeSigner = newSigner + c.lock.Unlock() +} + +func (c *CA) shutdownRefresh() { + close(c.shutdown) + <-c.done +} + +// IssuePEM issues a new server certificate for the given identity and duration, returning it as a pair of +// PEM-formatted byte slices for the certificate and private key. +func (c *CA) IssuePEM(subject pkix.Name, dnsNames []string, ttl time.Duration) ([]byte, []byte, error) { + c.lock.RLock() + signer := c.activeSigner + c.lock.RUnlock() + + return signer.IssuePEM(subject, dnsNames, ttl) +} + +func findControllerManagerPod(ctx context.Context, kubeClient kubernetes.Interface) (*v1.Pod, error) { + pods, err := kubeClient.CoreV1().Pods("kube-system").List(ctx, metav1.ListOptions{ + LabelSelector: "component=kube-controller-manager", + FieldSelector: "status.phase=Running", + }) + if err != nil { + return nil, fmt.Errorf("could not check for kube-controller-manager pod: %w", err) + } + for _, pod := range pods.Items { + return &pod, nil + } + return nil, ErrNoKubeControllerManagerPod +} + +func getKeypairFilePaths(pod *v1.Pod) (string, string) { + certPath := getContainerArgByName(pod, "cluster-signing-cert-file", k8sAPIServerCACertPEMDefaultPath) + keyPath := getContainerArgByName(pod, "cluster-signing-key-file", k8sAPIServerCAKeyPEMDefaultPath) + return certPath, keyPath +} + +func getContainerArgByName(pod *v1.Pod, name string, defaultValue string) string { + for _, container := range pod.Spec.Containers { + flagset := pflag.NewFlagSet("", pflag.ContinueOnError) + flagset.ParseErrorsWhitelist = pflag.ParseErrorsWhitelist{UnknownFlags: true} + var val string + flagset.StringVar(&val, name, "", "") + _ = flagset.Parse(append(container.Command, container.Args...)) + if val != "" { + return val + } + } + return defaultValue +} diff --git a/internal/certauthority/kubecertauthority/kubecertauthority_test.go b/internal/certauthority/kubecertauthority/kubecertauthority_test.go new file mode 100644 index 00000000..99c39055 --- /dev/null +++ b/internal/certauthority/kubecertauthority/kubecertauthority_test.go @@ -0,0 +1,329 @@ +/* +Copyright 2020 VMware, Inc. +SPDX-License-Identifier: Apache-2.0 +*/ + +package kubecertauthority + +import ( + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "errors" + "fmt" + "io/ioutil" + "testing" + "time" + + "github.com/sclevine/spec" + "github.com/sclevine/spec/report" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kubernetesfake "k8s.io/client-go/kubernetes/fake" + "k8s.io/klog/v2" + + "github.com/suzerain-io/placeholder-name/internal/testutil" +) + +type fakePodExecutor struct { + resultsToReturn []string + errorsToReturn []error + + calledWithPodName []string + calledWithPodNamespace []string + calledWithCommandAndArgs [][]string + + callCount int +} + +func (s *fakePodExecutor) Exec(podNamespace string, podName string, commandAndArgs ...string) (string, error) { + s.calledWithPodNamespace = append(s.calledWithPodNamespace, podNamespace) + s.calledWithPodName = append(s.calledWithPodName, podName) + s.calledWithCommandAndArgs = append(s.calledWithCommandAndArgs, commandAndArgs) + result := s.resultsToReturn[s.callCount] + var err error = nil + if s.errorsToReturn != nil { + err = s.errorsToReturn[s.callCount] + } + s.callCount++ + if err != nil { + return "", err + } + return result, nil +} + +func TestCA(t *testing.T) { + spec.Run(t, "CA", func(t *testing.T, when spec.G, it spec.S) { + var r *require.Assertions + var fakeCertPEM, fakeKeyPEM string + var fakeCert2PEM, fakeKey2PEM string + var fakePod *corev1.Pod + var kubeAPIClient *kubernetesfake.Clientset + var fakeExecutor *fakePodExecutor + var neverTicker <-chan time.Time + + var logger *testutil.TranscriptLogger + + it.Before(func() { + r = require.New(t) + + loadFile := func(filename string) string { + bytes, err := ioutil.ReadFile(filename) + r.NoError(err) + return string(bytes) + } + fakeCertPEM = loadFile("./testdata/test.crt") + fakeKeyPEM = loadFile("./testdata/test.key") + fakeCert2PEM = loadFile("./testdata/test2.crt") + fakeKey2PEM = loadFile("./testdata/test2.key") + + fakePod = &corev1.Pod{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{ + Name: "fake-pod", + Namespace: "kube-system", + Labels: map[string]string{"component": "kube-controller-manager"}, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{Name: "kube-controller-manager"}}, + }, + Status: corev1.PodStatus{ + Phase: "Running", + }, + } + + kubeAPIClient = kubernetesfake.NewSimpleClientset() + + fakeExecutor = &fakePodExecutor{ + resultsToReturn: []string{ + fakeCertPEM, + fakeKeyPEM, + fakeCert2PEM, + fakeKey2PEM, + }, + } + + logger = testutil.NewTranscriptLogger(t) + klog.SetLogger(logger) // this is unfortunately a global logger, so can't run these tests in parallel :( + }) + + it.After(func() { + klog.SetLogger(nil) + }) + + when("the kube-controller-manager pod is found with default CLI flag values", func() { + it.Before(func() { + err := kubeAPIClient.Tracker().Add(fakePod) + r.NoError(err) + }) + + when("the exec commands return the API server's keypair", func() { + it("finds the API server's signing key and uses it to issue certificates", func() { + fakeTicker := make(chan time.Time) + + subject, shutdownFunc, err := New(kubeAPIClient, fakeExecutor, fakeTicker) + r.NoError(err) + r.NotNil(shutdownFunc) + defer shutdownFunc() + + r.Equal(2, fakeExecutor.callCount) + + r.Equal("kube-system", fakeExecutor.calledWithPodNamespace[0]) + r.Equal("fake-pod", fakeExecutor.calledWithPodName[0]) + r.Equal([]string{"cat", "/etc/kubernetes/ca/ca.pem"}, fakeExecutor.calledWithCommandAndArgs[0]) + + r.Equal("kube-system", fakeExecutor.calledWithPodNamespace[1]) + r.Equal("fake-pod", fakeExecutor.calledWithPodName[1]) + r.Equal([]string{"cat", "/etc/kubernetes/ca/ca.key"}, fakeExecutor.calledWithCommandAndArgs[1]) + + // Validate that we can issue a certificate signed by the original API server CA. + certPEM, keyPEM, err := subject.IssuePEM( + pkix.Name{CommonName: "Test Server"}, + []string{"example.com"}, + 10*time.Minute, + ) + r.NoError(err) + validCert := testutil.ValidateCertificate(t, fakeCertPEM, string(certPEM)) + validCert.RequireDNSName("example.com") + validCert.RequireLifetime(time.Now(), time.Now().Add(10*time.Minute), 2*time.Minute) + validCert.RequireMatchesPrivateKey(string(keyPEM)) + + // Tick the timer and wait for another refresh loop to complete. + fakeTicker <- time.Now() + + var secondCertPEM, secondKeyPEM string + r.Eventually(func() bool { + certPEM, keyPEM, err := subject.IssuePEM( + pkix.Name{CommonName: "Test Server"}, + []string{"example.com"}, + 10*time.Minute, + ) + r.NoError(err) + secondCertPEM = string(certPEM) + secondKeyPEM = string(keyPEM) + + block, _ := pem.Decode(certPEM) + require.NotNil(t, block) + parsed, err := x509.ParseCertificate(block.Bytes) + require.NoError(t, err) + + // Validate the created cert using the second API server CA. + roots := x509.NewCertPool() + require.True(t, roots.AppendCertsFromPEM([]byte(fakeCert2PEM))) + opts := x509.VerifyOptions{Roots: roots} + _, err = parsed.Verify(opts) + return err == nil + }, 5*time.Second, 100*time.Millisecond) + + validCert2 := testutil.ValidateCertificate(t, fakeCert2PEM, secondCertPEM) + validCert2.RequireDNSName("example.com") + validCert2.RequireLifetime(time.Now(), time.Now().Add(10*time.Minute), 2*time.Minute) + validCert2.RequireMatchesPrivateKey(secondKeyPEM) + }) + }) + + when("the exec commands return the API server's keypair the first time but subsequently fails", func() { + it.Before(func() { + fakeExecutor.errorsToReturn = []error{nil, nil, fmt.Errorf("some exec error")} + }) + + it("logs an error message", func() { + fakeTicker := make(chan time.Time) + + subject, shutdownFunc, err := New(kubeAPIClient, fakeExecutor, fakeTicker) + r.NoError(err) + r.NotNil(shutdownFunc) + defer shutdownFunc() + r.Equal(2, fakeExecutor.callCount) + + // Tick the timer and wait for another refresh loop to complete. + fakeTicker <- time.Now() + + // Wait for there to be a log output and require that it matches our expectation. + r.Eventually(func() bool { return len(logger.Transcript()) >= 1 }, 5*time.Second, 10*time.Millisecond) + r.Contains(logger.Transcript()[0].Message, "could not create signer with API server secret: some exec error") + r.Equal(logger.Transcript()[0].Level, "error") + + // Validate that we can still issue a certificate signed by the original API server CA. + certPEM, _, err := subject.IssuePEM( + pkix.Name{CommonName: "Test Server"}, + []string{"example.com"}, + 10*time.Minute, + ) + r.NoError(err) + testutil.ValidateCertificate(t, fakeCertPEM, string(certPEM)) + }) + }) + + when("the exec commands succeed but return garbage", func() { + it.Before(func() { + fakeExecutor.resultsToReturn = []string{"not a cert", "not a private key"} + }) + + it("returns an error", func() { + subject, shutdownFunc, err := New(kubeAPIClient, fakeExecutor, neverTicker) + r.Nil(subject) + r.Nil(shutdownFunc) + r.EqualError(err, "could not load CA: tls: failed to find any PEM data in certificate input") + }) + }) + + when("the first exec command returns an error", func() { + it.Before(func() { + fakeExecutor.errorsToReturn = []error{fmt.Errorf("some error"), nil} + }) + + it("returns an error", func() { + subject, shutdownFunc, err := New(kubeAPIClient, fakeExecutor, neverTicker) + r.Nil(subject) + r.Nil(shutdownFunc) + r.EqualError(err, "some error") + }) + }) + + when("the second exec command returns an error", func() { + it.Before(func() { + fakeExecutor.errorsToReturn = []error{nil, fmt.Errorf("some error")} + }) + + it("returns an error", func() { + subject, shutdownFunc, err := New(kubeAPIClient, fakeExecutor, neverTicker) + r.Nil(subject) + r.Nil(shutdownFunc) + r.EqualError(err, "some error") + }) + }) + }) + + when("the kube-controller-manager pod is found with non-default CLI flag values", func() { + it.Before(func() { + fakePod.Spec.Containers[0].Command = []string{ + "kube-controller-manager", + "--cluster-signing-cert-file=/etc/kubernetes/ca/non-default.pem", + } + fakePod.Spec.Containers[0].Args = []string{ + "--cluster-signing-key-file=/etc/kubernetes/ca/non-default.key", + } + err := kubeAPIClient.Tracker().Add(fakePod) + r.NoError(err) + }) + + it("finds the API server's signing key and uses it to issue certificates", func() { + _, shutdownFunc, err := New(kubeAPIClient, fakeExecutor, neverTicker) + r.NoError(err) + r.NotNil(shutdownFunc) + defer shutdownFunc() + + r.Equal(2, fakeExecutor.callCount) + + r.Equal("kube-system", fakeExecutor.calledWithPodNamespace[0]) + r.Equal("fake-pod", fakeExecutor.calledWithPodName[0]) + r.Equal([]string{"cat", "/etc/kubernetes/ca/non-default.pem"}, fakeExecutor.calledWithCommandAndArgs[0]) + + r.Equal("kube-system", fakeExecutor.calledWithPodNamespace[1]) + r.Equal("fake-pod", fakeExecutor.calledWithPodName[1]) + r.Equal([]string{"cat", "/etc/kubernetes/ca/non-default.key"}, fakeExecutor.calledWithCommandAndArgs[1]) + }) + }) + + when("the kube-controller-manager pod is found with non-default CLI flag values separated by spaces", func() { + it.Before(func() { + fakePod.Spec.Containers[0].Command = []string{ + "kube-controller-manager", + "--cluster-signing-cert-file", "/etc/kubernetes/ca/non-default.pem", + "--cluster-signing-key-file", "/etc/kubernetes/ca/non-default.key", + "--foo=bar", + } + err := kubeAPIClient.Tracker().Add(fakePod) + r.NoError(err) + }) + + it("finds the API server's signing key and uses it to issue certificates", func() { + _, shutdownFunc, err := New(kubeAPIClient, fakeExecutor, neverTicker) + r.NoError(err) + r.NotNil(shutdownFunc) + defer shutdownFunc() + + r.Equal(2, fakeExecutor.callCount) + + r.Equal("kube-system", fakeExecutor.calledWithPodNamespace[0]) + r.Equal("fake-pod", fakeExecutor.calledWithPodName[0]) + r.Equal([]string{"cat", "/etc/kubernetes/ca/non-default.pem"}, fakeExecutor.calledWithCommandAndArgs[0]) + + r.Equal("kube-system", fakeExecutor.calledWithPodNamespace[1]) + r.Equal("fake-pod", fakeExecutor.calledWithPodName[1]) + r.Equal([]string{"cat", "/etc/kubernetes/ca/non-default.key"}, fakeExecutor.calledWithCommandAndArgs[1]) + }) + }) + + when("the kube-controller-manager pod is not found", func() { + it("returns an error", func() { + subject, shutdownFunc, err := New(kubeAPIClient, fakeExecutor, neverTicker) + r.Nil(subject) + r.Nil(shutdownFunc) + r.True(errors.Is(err, ErrNoKubeControllerManagerPod)) + }) + }) + }, spec.Report(report.Terminal{})) +} diff --git a/internal/certauthority/kubecertauthority/testdata/test.crt b/internal/certauthority/kubecertauthority/testdata/test.crt new file mode 100644 index 00000000..796a7690 --- /dev/null +++ b/internal/certauthority/kubecertauthority/testdata/test.crt @@ -0,0 +1,17 @@ +-----BEGIN CERTIFICATE----- +MIICyDCCAbCgAwIBAgIBADANBgkqhkiG9w0BAQsFADAVMRMwEQYDVQQDEwprdWJl +cm5ldGVzMB4XDTIwMDcyNTIxMDQxOFoXDTMwMDcyMzIxMDQxOFowFTETMBEGA1UE +AxMKa3ViZXJuZXRlczCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAL3K +hYv2gIQ1Dwzh2cWMid+ofAnvLIfV2Xv61vTLGprUI+XUqB4/gtf6X6UNn0Lett2n +d8p4wy7hw73hU/ggdvmWJvqBrSjc3JGfy+kj66fKXX+PTlbL7QbwiRvcSqIXIWlV +lHHxECWrED8jCulw/NVqfook/h5iNUCT9yswSJr/0fImiVnoTlIoEYG2eCNejZ5c +g39uD3ZTqd9ZxWwSLLnI+2kpJnZBPcd1ZQ8AQqzDgZtYRCqacn5gckQUKZWKQlxo +Eft6g1XHJouAWAZw7hEtk0v8rG0/eKF7wamxFi6BFVlbjWBsB4T9rApbdBWTKeCJ +Hv8fv5RMFSzpT3uzTO8CAwEAAaMjMCEwDgYDVR0PAQH/BAQDAgKkMA8GA1UdEwEB +/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBACh5RhbxqJe+Z/gc17cZhKNmdiwu +I2pLp3QBfwvN+Wbmajzw/7rYhY0d8JYVTJzXSCPWi6UAKxAtXOLF8WIIf9i39n6R +uKOBGW14FzzGyRJiD3qaG/JTvEW+SLhwl68Ndr5LHSnbugAqq31abcQy6Zl9v5A8 +JKC97Lj/Sn8rj7opKy4W3oq7NCQsAb0zh4IllRF6UvSnJySfsg7xdXHHpxYDHtOS +XcOu5ySUIZTgFe9RfeUZlGZ5xn0ckMlQ7qW2Wx1q0OVWw5us4NtkGqKrHG4Tn1X7 +uwo/Yytn5sDxrDv1/oii6AZOCsTPre4oD3wz4nmVzCVJcgrqH4Q24hT8WNg= +-----END CERTIFICATE----- diff --git a/internal/certauthority/kubecertauthority/testdata/test.key b/internal/certauthority/kubecertauthority/testdata/test.key new file mode 100644 index 00000000..7ad653ae --- /dev/null +++ b/internal/certauthority/kubecertauthority/testdata/test.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEogIBAAKCAQEAvcqFi/aAhDUPDOHZxYyJ36h8Ce8sh9XZe/rW9MsamtQj5dSo +Hj+C1/pfpQ2fQt623ad3ynjDLuHDveFT+CB2+ZYm+oGtKNzckZ/L6SPrp8pdf49O +VsvtBvCJG9xKohchaVWUcfEQJasQPyMK6XD81Wp+iiT+HmI1QJP3KzBImv/R8iaJ +WehOUigRgbZ4I16NnlyDf24PdlOp31nFbBIsucj7aSkmdkE9x3VlDwBCrMOBm1hE +KppyfmByRBQplYpCXGgR+3qDVccmi4BYBnDuES2TS/ysbT94oXvBqbEWLoEVWVuN +YGwHhP2sClt0FZMp4Ike/x+/lEwVLOlPe7NM7wIDAQABAoIBAFC1tUEmHNUcM0BJ +M3D9KQzB+63F1mwVlx1QOOV1EeVR3co5Ox1R6PSr9sycFGQ9jgqI0zp5TJe9Tp6L +GkhklfPh1MWnK9o6wlnzWKXWrrp2Jni+mpPyuOPAmq4Maniv2XeP+0bROwqpyojv +AA7yC7M+TH226ZJGNVs3EV9+cwHml0yuzBfIJn/rv/w2g+WRKM/MC0S7k2d8bRlA +NycKVGAGBhKTltjoVYOeh6aHEpSjK8zfaePjo5dYJvoVIli60YCgcJOU/8jXT+Np +1Fm7tRvAtj3pUp0Sqdaf2RUzh9jfJp2VFCHuSJ6TPqArOyQojtMcTHF0TiW7xrHP +xOCRIAECgYEAwGBPU7vdthMJBg+ORUoGQQaItTeJvQwIqJvbKD2osp4jhS1dGZBw +W30GKEc/gd8JNtOq9BBnMicPF7hktuy+bSPv41XPud67rSSO7Tsw20C10gFRq06B +zIJWFAUqK3IkvVc3VDmtSLSDox4QZ/BdqaMlQ5y5JCsC5kThmkZFlO8CgYEA/I9X +YHi6RioMJE1fqOHJL4DDjlezmcuRrD7fE5InKbtJZ2JhGYOX/C0KXnHTOWTCDxxN +FBvpvD6Xv5o3PhB9Z6k2fqvJ4GS8urkG/KU4xcC+bak+9ava8oaiSqG16zD9NH2P +jJ60NrbLl1J0pU9fiwuFVUKJ4hDZOfN9RqYdyAECgYAVwo8WhJiGgM6zfcz073OX +pVqPTPHqjVLpZ3+5pIfRdGvGI6R1QM5EuvaYVb7MPOM47WZX5wcVOC/P2g6iVlMP +21HGIC2384a9BfaYxOo40q/+SiHnw6CQ9mkwKIllkqqvNA9RGpkMMUb2i28For2l +c4vCgxa6DZdtXns6TRqPxwKBgCfY5cxOv/T6BVhk7MbUeM2J31DB/ZAyUhV/Bess +kAlBh19MYk2IOZ6L7KriApV3lDaWHIMjtEkDByYvyq98Io0MYZCywfMpca10K+oI +l2B7/I+IuGpCZxUEsO5dfTpSTGDPvqpND9niFVUWqVi7oTNq6ep9yQtl5SADjqxq +4SABAoGAIm0hUg1wtcS46cGLy6PIkPM5tocTSghtz4vFsuk/i4QA9GBoBO2gH6ty ++kJHmeaXt2dmgySp0QAWit5UlceEumB0NXnAdJZQxeGSFSyYkDWhwXd8wDceKo/1 +LfCU6Dk8IN/SsppVUWXQ2rlORvxlrHeCio8o0kS9Yiu55WMYg4g= +-----END RSA PRIVATE KEY----- diff --git a/internal/certauthority/kubecertauthority/testdata/test2.crt b/internal/certauthority/kubecertauthority/testdata/test2.crt new file mode 100644 index 00000000..108e6e00 --- /dev/null +++ b/internal/certauthority/kubecertauthority/testdata/test2.crt @@ -0,0 +1,17 @@ +-----BEGIN CERTIFICATE----- +MIICyDCCAbCgAwIBAgIBADANBgkqhkiG9w0BAQsFADAVMRMwEQYDVQQDEwprdWJl +cm5ldGVzMB4XDTIwMDgxODE2NDEzNloXDTMwMDgxNjE2NDEzNlowFTETMBEGA1UE +AxMKa3ViZXJuZXRlczCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALH7 +C2JpttDi3mxpD4bd+BZucCrS8XF2YwqYAr42HePp++PBnlUFqWmtPc9/bmo+7+7z +iAAlnAV0pJWP+HR/PskX8MRcFAA1HoXLa37Q4SuBBQG+JE+AeaOObmQYaCFv55ej +UF4+JIoQOdlbYEMYSI07el0cxQL4Io/CHJ3p7AtNElxjDuMK4B9W8NiCse3p7Uf+ +Qje4we1TYOfcpAM0jpBPHK9vCBCpX+j52S5DUTRVIk9kye3lCDmWOXH/fhj/aJTM +1MP/hThbl2wIbFuv1bpa0kXNZs8xB63dtqROQ+lCghDmuayRmzwXl2PX6IgFFcjV +yAgjXrZqjihs+mY8eT0CAwEAAaMjMCEwDgYDVR0PAQH/BAQDAgKkMA8GA1UdEwEB +/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAE+Saqk2EyuIx1rxFWrOwpTi5q/B +p/TwEtrmrFIRVPnGeBnhyfbGXPDMkzIY1mEvztu8H+pm5RPyhQYLsuwzYiYMQyxX +yL9VvO7uydn7+3zX7oknQ5qAvN3nmItNyOKw3MRIKGsySNuTQ5JPtU/ufGlEivbK +vNaDBqjKrBvwhIKMdV9/xYSyeBhSSWr/6W1tAk+XbHhQH1M78rdwGN5SI75L4FGu +13kn/W2n8pE17TAY88B1YGKhsLSvf8KrFNYv+UUmzh2WstECKSlnbrSM+boMlGJn +XahE8M23fieB+SaenQdOezrY4GAnXQ3qToDlhdYAOkWhcGDct47VRM93whY= +-----END CERTIFICATE----- diff --git a/internal/certauthority/kubecertauthority/testdata/test2.key b/internal/certauthority/kubecertauthority/testdata/test2.key new file mode 100644 index 00000000..951d1c99 --- /dev/null +++ b/internal/certauthority/kubecertauthority/testdata/test2.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEogIBAAKCAQEAsfsLYmm20OLebGkPht34Fm5wKtLxcXZjCpgCvjYd4+n748Ge +VQWpaa09z39uaj7v7vOIACWcBXSklY/4dH8+yRfwxFwUADUehctrftDhK4EFAb4k +T4B5o45uZBhoIW/nl6NQXj4kihA52VtgQxhIjTt6XRzFAvgij8IcnensC00SXGMO +4wrgH1bw2IKx7entR/5CN7jB7VNg59ykAzSOkE8cr28IEKlf6PnZLkNRNFUiT2TJ +7eUIOZY5cf9+GP9olMzUw/+FOFuXbAhsW6/VulrSRc1mzzEHrd22pE5D6UKCEOa5 +rJGbPBeXY9foiAUVyNXICCNetmqOKGz6Zjx5PQIDAQABAoIBAD06klYO7De8dKxz +EEZjgn+lCq2Q2EMiaTwxw2/QikPoMSHPcDrrsbaLROJngoLGmCBqY3U5ew1dbWmO +l/jr9ZuUwt2ql67il1eL/bUpAu3GewR4d2FqX25nB48j3l7ycof2RSXG1ycwIdam +2tz6M6tytMvno9c7qhguvU2ONghEreXG3YYLdf9l97aB+p6GdXhwty22b7tAVwp1 +GKn79kVYgmL86lph9hBPqtHuG1LHZUiFodr2iWXSu3H/265OD58a33ZO3iyfFI0s +PPy87ZN0r+1hGpoKKkDe63udOYgAG6xmIea/1Pdn9Eg87tueoeC7XcUpdaCJlKaF +tqCusEECgYEA60rWyXxTFKJ4QdVaqXoWMA4cQkT73RxznSKwkN/Svk8TVv+p5s5Y +oYKN4qyMzxvQzu+QNWpd1yTveCmmEynz457ELpGtidtiJdm7xZMdMGrU02eCL9mZ +ERbtfAkbEAKvN8D73fWyzghKv4dgcQptmsqZlYYc4vpwHveK+/N5lukCgYEAwaT3 +iMTWCv7Vp87iKrzNUAH4iBWlazwbE+EDEnHVw26Y82fhgEgxiU2UvFSaIVhGpaCz +MYSXSdRcQTHgCoJLPfWHUHTJPqf36KfAJfdaxxjzTTbNYjUOkdcUD1bcNrm0yjoY +nR4zK1FPw86ODMYtBpfkyL7ZX8G1v5pRL/6/gzUCgYBzgwQ7Wmu3H6QGPeYKecNW +yDabWh6ECKnBpPwlw5xEjbGi7lTM2NSuRde+RpPCQZebYATeFGAJdTqTNW8wzVHM +l28cpawal7dxeZkzf+u+j1P4jUJel2cL+sOQNzAwBgFbT8TWzP6BI5T+vklcdZAl +g/0uaO7Zh7Vvnnt/AaLZsQKBgGfbHzuGPjoFdPecOKatPfxUIkRyP5bk1KzzuF8T +GI/JaFTbeREBJzg5mLTtNwD9RF6ecpzzPOTG9Xet1Tgtq0cewSUAjdKB6a8pESAL +qu8vTYYzBzJNvHOxg7u6XT8omHMBd6QEx3LLGFmvFXZ6bzmjC3wzB4iY7u5FSJfS +LEqlAoGAb0rbJ85vrJopbx8lzhJjyaJfM8+A3oQg1K3TphHrcgkUc8qx8QEosziM +wzYKSBlXd2bxMibyd0mTEMNl4/BqofaKoqof9gBIbkamwXOO8s7IgKxQAfr1R/z8 +tHBW/g0QWPB+qtaVDtHwyQLlxjx8HD7atIo8d/do9ruwVaf+r6g= +-----END RSA PRIVATE KEY----- diff --git a/internal/controller/apicerts/certs_manager_test.go b/internal/controller/apicerts/certs_manager_test.go index 6e2fe36c..decdc0b9 100644 --- a/internal/controller/apicerts/certs_manager_test.go +++ b/internal/controller/apicerts/certs_manager_test.go @@ -7,9 +7,6 @@ package apicerts import ( "context" - "crypto/tls" - "crypto/x509" - "encoding/pem" "errors" "testing" "time" @@ -222,29 +219,10 @@ func TestManagerControllerSync(t *testing.T) { r.NotEmpty(actualCertChain) // Validate the created cert using the CA, and also validate the cert's hostname - roots := x509.NewCertPool() - ok := roots.AppendCertsFromPEM([]byte(actualCACert)) - r.True(ok) - block, _ := pem.Decode([]byte(actualCertChain)) - r.NotNil(block) - parsedCert, err := x509.ParseCertificate(block.Bytes) - r.NoError(err) - serviceEndpoint := "placeholder-name-api." + installedInNamespace + ".svc" - opts := x509.VerifyOptions{ - DNSName: serviceEndpoint, - Roots: roots, - } - _, err = parsedCert.Verify(opts) - r.NoError(err) - r.Contains(parsedCert.DNSNames, serviceEndpoint, "expected an explicit DNS SAN, not just Common Name") - - // Check the created cert's validity bounds - r.WithinDuration(time.Now(), parsedCert.NotBefore, time.Minute*2) - r.WithinDuration(time.Now().Add(24*365*time.Hour), parsedCert.NotAfter, time.Minute*2) - - // Check that the private key and cert chain match - _, err = tls.X509KeyPair([]byte(actualCertChain), []byte(actualPrivateKey)) - r.NoError(err) + validCert := testutil.ValidateCertificate(t, actualCACert, actualCertChain) + validCert.RequireDNSName("placeholder-name-api." + installedInNamespace + ".svc") + validCert.RequireLifetime(time.Now(), time.Now().Add(24*365*time.Hour), 2*time.Minute) + validCert.RequireMatchesPrivateKey(actualPrivateKey) // Make sure we updated the APIService caBundle and left it otherwise unchanged r.Len(aggregatorAPIClient.Actions(), 2) diff --git a/internal/controllermanager/prepare_controllers.go b/internal/controllermanager/prepare_controllers.go index eee8d954..a5d87e85 100644 --- a/internal/controllermanager/prepare_controllers.go +++ b/internal/controllermanager/prepare_controllers.go @@ -97,7 +97,7 @@ func createClients() ( placeholderClient *placeholderclientset.Clientset, err error, ) { - // Load the Kubernetes client configuration (kubeconfig), + // Load the Kubernetes client configuration. kubeConfig, err := restclient.InClusterConfig() if err != nil { return nil, nil, nil, fmt.Errorf("could not load in-cluster configuration: %w", err) diff --git a/internal/registry/credentialrequest/rest_test.go b/internal/registry/credentialrequest/rest_test.go index 837b4df8..a23d0d19 100644 --- a/internal/registry/credentialrequest/rest_test.go +++ b/internal/registry/credentialrequest/rest_test.go @@ -425,9 +425,10 @@ func TestCreate(t *testing.T) { } func requireOneLogStatement(r *require.Assertions, logger *testutil.TranscriptLogger, messageContains string) { - r.Len(logger.Transcript, 1) - r.Equal("info", logger.Transcript[0].Level) - r.Contains(logger.Transcript[0].Message, messageContains) + transcript := logger.Transcript() + r.Len(transcript, 1) + r.Equal("info", transcript[0].Level) + r.Contains(transcript[0].Message, messageContains) } func callCreate(ctx context.Context, storage *REST, credentialRequest *placeholderapi.CredentialRequest) (runtime.Object, error) { diff --git a/internal/server/server.go b/internal/server/server.go index 9711c6f8..3576ce0e 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -10,17 +10,21 @@ import ( "context" "fmt" "io" + "time" "github.com/spf13/cobra" genericapiserver "k8s.io/apiserver/pkg/server" genericoptions "k8s.io/apiserver/pkg/server/options" "k8s.io/apiserver/plugin/pkg/authenticator/token/webhook" + "k8s.io/client-go/kubernetes" + restclient "k8s.io/client-go/rest" "github.com/suzerain-io/placeholder-name/internal/apiserver" - "github.com/suzerain-io/placeholder-name/internal/certauthority" + "github.com/suzerain-io/placeholder-name/internal/certauthority/kubecertauthority" "github.com/suzerain-io/placeholder-name/internal/controllermanager" "github.com/suzerain-io/placeholder-name/internal/downward" "github.com/suzerain-io/placeholder-name/internal/provider" + "github.com/suzerain-io/placeholder-name/internal/registry/credentialrequest" placeholderv1alpha1 "github.com/suzerain-io/placeholder-name/kubernetes/1.19/api/apis/placeholder/v1alpha1" "github.com/suzerain-io/placeholder-name/pkg/config" ) @@ -30,10 +34,8 @@ type App struct { cmd *cobra.Command // CLI flags - configPath string - downwardAPIPath string - clusterSigningCertFilePath string - clusterSigningKeyFilePath string + configPath string + downwardAPIPath string } // This is ignored for now because we turn off etcd storage below, but this is @@ -87,20 +89,6 @@ func addCommandlineFlagsToCommand(cmd *cobra.Command, app *App) { "/etc/podinfo", "path to Downward API volume mount", ) - - cmd.Flags().StringVar( - &app.clusterSigningCertFilePath, - "cluster-signing-cert-file", - "", - "path to cluster signing certificate", - ) - - cmd.Flags().StringVar( - &app.clusterSigningKeyFilePath, - "cluster-signing-key-file", - "", - "path to cluster signing private key", - ) } // Boot the aggregated API server, which will in turn boot the controllers. @@ -112,10 +100,11 @@ func (a *App) runServer(ctx context.Context) error { } // Load the Kubernetes cluster signing CA. - k8sClusterCA, err := certauthority.Load(a.clusterSigningCertFilePath, a.clusterSigningKeyFilePath) + k8sClusterCA, shutdownCA, err := getClusterCASigner() if err != nil { - return fmt.Errorf("could not load cluster signing CA: %w", err) + return err } + defer shutdownCA() // Create a WebhookTokenAuthenticator. webhookTokenAuthenticator, err := config.NewWebhook(cfg.WebhookConfig) @@ -169,11 +158,40 @@ func (a *App) runServer(ctx context.Context) error { return server.GenericAPIServer.PrepareRun().Run(ctx.Done()) } +func getClusterCASigner() (*kubecertauthority.CA, kubecertauthority.ShutdownFunc, error) { + // Load the Kubernetes client configuration. + kubeConfig, err := restclient.InClusterConfig() + if err != nil { + return nil, nil, fmt.Errorf("could not load in-cluster configuration: %w", err) + } + + // Connect to the core Kubernetes API. + kubeClient, err := kubernetes.NewForConfig(kubeConfig) + if err != nil { + return nil, nil, fmt.Errorf("could not initialize Kubernetes client: %w", err) + } + + // Make a clock tick that triggers a periodic refresh. + ticker := time.NewTicker(5 * time.Minute) + + // Make a CA which uses the Kubernetes cluster API server's signing certs. + k8sClusterCA, shutdownCA, err := kubecertauthority.New( + kubeClient, + kubecertauthority.NewPodCommandExecutor(kubeConfig, kubeClient), + ticker.C, + ) + if err != nil { + ticker.Stop() + return nil, nil, fmt.Errorf("could not load cluster signing CA: %w", err) + } + return k8sClusterCA, func() { shutdownCA(); ticker.Stop() }, nil +} + // Create a configuration for the aggregated API server. func getAggregatedAPIServerConfig( dynamicCertProvider provider.DynamicTLSServingCertProvider, webhookTokenAuthenticator *webhook.WebhookTokenAuthenticator, - ca *certauthority.CA, + ca credentialrequest.CertIssuer, startControllersPostStartHook func(context.Context), ) (*apiserver.Config, error) { recommendedOptions := genericoptions.NewRecommendedOptions( diff --git a/internal/server/server_test.go b/internal/server/server_test.go index 5ae9abe8..6bd64214 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -25,12 +25,10 @@ Usage: placeholder-name-server [flags] Flags: - --cluster-signing-cert-file string path to cluster signing certificate - --cluster-signing-key-file string path to cluster signing private key - -c, --config string path to configuration file (default "placeholder-name.yaml") - --downward-api-path string path to Downward API volume mount (default "/etc/podinfo") - -h, --help help for placeholder-name-server - --log-flush-frequency duration Maximum number of seconds between log flushes (default 5s) + -c, --config string path to configuration file (default "placeholder-name.yaml") + --downward-api-path string path to Downward API volume mount (default "/etc/podinfo") + -h, --help help for placeholder-name-server + --log-flush-frequency duration Maximum number of seconds between log flushes (default 5s) ` func TestCommand(t *testing.T) { diff --git a/internal/testutil/certs.go b/internal/testutil/certs.go new file mode 100644 index 00000000..932e3762 --- /dev/null +++ b/internal/testutil/certs.go @@ -0,0 +1,73 @@ +/* +Copyright 2020 VMware, Inc. +SPDX-License-Identifier: Apache-2.0 +*/ + +package testutil + +import ( + "crypto/tls" + "crypto/x509" + "encoding/pem" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +type ValidCert struct { + t *testing.T + roots *x509.CertPool + certPEM string + parsed *x509.Certificate +} + +// ValidateCertificate validates a certificate and provides an object for asserting properties of the certificate. +func ValidateCertificate(t *testing.T, caPEM string, certPEM string) *ValidCert { + t.Helper() + + block, _ := pem.Decode([]byte(certPEM)) + require.NotNil(t, block) + parsed, err := x509.ParseCertificate(block.Bytes) + require.NoError(t, err) + + // Validate the created cert using the CA. + roots := x509.NewCertPool() + require.True(t, roots.AppendCertsFromPEM([]byte(caPEM))) + opts := x509.VerifyOptions{Roots: roots} + _, err = parsed.Verify(opts) + require.NoError(t, err) + + return &ValidCert{ + t: t, + roots: roots, + certPEM: certPEM, + parsed: parsed, + } +} + +// RequireDNSName asserts that the certificate matches the provided DNS name. +func (v *ValidCert) RequireDNSName(expectDNSName string) { + v.t.Helper() + opts := x509.VerifyOptions{ + Roots: v.roots, + DNSName: expectDNSName, + } + _, err := v.parsed.Verify(opts) + require.NoError(v.t, err) + require.Contains(v.t, v.parsed.DNSNames, expectDNSName, "expected an explicit DNS SAN, not just Common Name") +} + +// RequireLifetime asserts that the lifetime of the certificate matches the expected timestamps. +func (v *ValidCert) RequireLifetime(expectNotBefore time.Time, expectNotAfter time.Time, delta time.Duration) { + v.t.Helper() + require.WithinDuration(v.t, expectNotBefore, v.parsed.NotBefore, delta) + require.WithinDuration(v.t, expectNotAfter, v.parsed.NotAfter, delta) +} + +// RequireMatchesPrivateKey asserts that the public key in the certificate matches the provided private key. +func (v *ValidCert) RequireMatchesPrivateKey(keyPEM string) { + v.t.Helper() + _, err := tls.X509KeyPair([]byte(v.certPEM), []byte(keyPEM)) + require.NoError(v.t, err) +} diff --git a/internal/testutil/transcript_logger.go b/internal/testutil/transcript_logger.go index 2bf99e7b..41bd4d26 100644 --- a/internal/testutil/transcript_logger.go +++ b/internal/testutil/transcript_logger.go @@ -7,6 +7,7 @@ package testutil import ( "fmt" + "sync" "testing" "github.com/go-logr/logr" @@ -14,7 +15,8 @@ import ( type TranscriptLogger struct { t *testing.T - Transcript []TranscriptLogMessage + lock sync.Mutex + transcript []TranscriptLogMessage } var _ logr.Logger = &TranscriptLogger{} @@ -28,15 +30,27 @@ func NewTranscriptLogger(t *testing.T) *TranscriptLogger { return &TranscriptLogger{t: t} } +func (log *TranscriptLogger) Transcript() []TranscriptLogMessage { + log.lock.Lock() + defer log.lock.Unlock() + result := make([]TranscriptLogMessage, 0, len(log.transcript)) + result = append(result, log.transcript...) + return result +} + func (log *TranscriptLogger) Info(msg string, keysAndValues ...interface{}) { - log.Transcript = append(log.Transcript, TranscriptLogMessage{ + log.lock.Lock() + defer log.lock.Unlock() + log.transcript = append(log.transcript, TranscriptLogMessage{ Level: "info", Message: fmt.Sprintf(msg, keysAndValues...), }) } func (log *TranscriptLogger) Error(err error, msg string, keysAndValues ...interface{}) { - log.Transcript = append(log.Transcript, TranscriptLogMessage{ + log.lock.Lock() + defer log.lock.Unlock() + log.transcript = append(log.transcript, TranscriptLogMessage{ Level: "error", Message: fmt.Sprintf("%s: %v -- %v", msg, err, keysAndValues), }) diff --git a/test/integration/app_availability_test.go b/test/integration/app_availability_test.go index 6f830b16..e636baab 100644 --- a/test/integration/app_availability_test.go +++ b/test/integration/app_availability_test.go @@ -11,23 +11,39 @@ import ( "time" "github.com/stretchr/testify/require" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/suzerain-io/placeholder-name/test/library" ) -func TestAppAvailability(t *testing.T) { +func TestGetDeployment(t *testing.T) { library.SkipUnlessIntegration(t) namespaceName := library.Getenv(t, "PLACEHOLDER_NAME_NAMESPACE") - daemonSetName := library.Getenv(t, "PLACEHOLDER_NAME_APP_NAME") + deploymentName := library.Getenv(t, "PLACEHOLDER_NAME_APP_NAME") client := library.NewClientset(t) ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() - daemonSet, err := client.AppsV1().DaemonSets(namespaceName).Get(ctx, daemonSetName, metav1.GetOptions{}) + appDeployment, err := client.AppsV1().Deployments(namespaceName).Get(ctx, deploymentName, metav1.GetOptions{}) require.NoError(t, err) - require.GreaterOrEqual(t, daemonSet.Status.NumberAvailable, int32(1)) + cond := getDeploymentCondition(appDeployment.Status, appsv1.DeploymentAvailable) + require.NotNil(t, cond) + require.Equalf(t, corev1.ConditionTrue, cond.Status, "app should be available: %s", library.Sdump(appDeployment)) +} + +// getDeploymentCondition returns the condition with the provided type. +// Copied from k8s.io/kubectl/pkg/util/deployment/deployment.go to prevent us from vendoring the world. +func getDeploymentCondition(status appsv1.DeploymentStatus, condType appsv1.DeploymentConditionType) *appsv1.DeploymentCondition { + for i := range status.Conditions { + c := status.Conditions[i] + if c.Type == condType { + return &c + } + } + return nil }