djpbessems/ContainerImage.Pinniped
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Switch back to an exec-based approach to grab the controller-manager CA. (#65)

Browse Source 
This switches us back to an approach where we use the Pod "exec" API to grab the keys we need, rather than forcing our code to run on the control plane node. It will help us fail gracefully (or dynamically switch to alternate implementations) when the cluster is not self-hosted.

Signed-off-by: Matt Moyer <moyerm@vmware.com>
Co-authored-by: Ryan Richard <richardry@vmware.com>
This commit is contained in:
Matt Moyer 2020-08-19 13:21:07 -05:00 committed by GitHub
parent 40d1360b74
commit 1b9a70d089
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
20 changed files with 842 additions and 106 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
deploy/deployment.yaml
View File

@ -46,13 +46,14 @@ data:
#! TODO set up healthy, ready, etc. probes correctly?
#! TODO set resource minimums (e.g. 512MB RAM) to make sure we get scheduled onto a reasonable node?
apiVersion: apps/v1
kind: DaemonSet
kind: Deployment
metadata:
name: #@ data.values.app_name
namespace: #@ data.values.namespace
labels:
app: #@ data.values.app_name
spec:
replicas: 2
selector:
matchLabels:
app: #@ data.values.app_name
@ -79,15 +80,11 @@ spec:
args:
- --config=/etc/config/placeholder-name.yaml
- --downward-api-path=/etc/podinfo
- --cluster-signing-cert-file=/etc/kubernetes/pki/ca.crt
- --cluster-signing-key-file=/etc/kubernetes/pki/ca.key
volumeMounts:
- name: config-volume
mountPath: /etc/config
- name: podinfo
mountPath: /etc/podinfo
- name: k8s-certs
mountPath: /etc/kubernetes/pki
livenessProbe:
httpGet:
path: /healthz
@ -119,16 +116,10 @@ spec:
- path: "namespace"
fieldRef:
fieldPath: metadata.namespace
- name: k8s-certs
hostPath:
path: /etc/kubernetes/pki
type: DirectoryOrCreate
nodeSelector: #! Create Pods on all nodes which match this node selector, and not on any other nodes.
node-role.kubernetes.io/master: ""
tolerations:
- key: CriticalAddonsOnly
operator: Exists
- key: node-role.kubernetes.io/master #! Allow running on master nodes.
- key: node-role.kubernetes.io/master #! Allow running on master nodes too
effect: NoSchedule
#! "system-cluster-critical" cannot be used outside the kube-system namespace until Kubernetes >= 1.17,
#! so we skip setting this for now (see https://github.com/kubernetes/kubernetes/issues/60596).

29
deploy/rbac.yaml
View File

@ -62,6 +62,35 @@ roleRef:
name: #@ data.values.app_name + "-aggregated-api-server-role"
apiGroup: rbac.authorization.k8s.io
#! Give permission to list pods and pod exec in the kube-system namespace so we can find the API server's private key
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: #@ data.values.app_name + "-kube-system-pod-exec-role"
namespace: kube-system
rules:
- apiGroups: [""]
resources: [pods]
verbs: [get, list]
- apiGroups: [""]
resources: [pods/exec]
verbs: [create]
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: #@ data.values.app_name + "-kube-system-pod-exec-role-binding"
namespace: kube-system
subjects:
- kind: ServiceAccount
name: #@ data.values.app_name + "-service-account"
namespace: #@ data.values.namespace
roleRef:
kind: Role
name: #@ data.values.app_name + "-kube-system-pod-exec-role"
apiGroup: rbac.authorization.k8s.io
#! Allow both authenticated and unauthenticated CredentialRequests (i.e. allow all requests)
---
apiVersion: rbac.authorization.k8s.io/v1

1
go.mod
View File

@ -9,6 +9,7 @@ require (
github.com/google/go-cmp v0.5.0
github.com/sclevine/spec v1.4.0
github.com/spf13/cobra v1.0.0
github.com/spf13/pflag v1.0.5
github.com/stretchr/testify v1.6.1
github.com/suzerain-io/controller-go v0.0.0-20200730212956-7f99b569ca9f
github.com/suzerain-io/placeholder-name/kubernetes/1.19/api v0.0.0-00010101000000-000000000000

1
go.sum
View File

@ -100,6 +100,7 @@ github.com/denis-tingajkin/go-header v0.3.1/go.mod h1:sq/2IxMhaZX+RRcgHfCRx/m0M5
github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96 h1:cenwrSVm+Z7QLSV/BsnenAOcDXdX4cMv4wP0B/5QbPg=
github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=

4
internal/certauthority/certauthority.go
View File

@ -62,8 +62,8 @@ func secureEnv() env {
var ErrInvalidCACertificate = fmt.Errorf("invalid CA certificate")
// Load a certificate authority from an existing certificate and private key (in PEM format).
func Load(certPath string, keyPath string) (*CA, error) {
cert, err := tls.LoadX509KeyPair(certPath, keyPath)
func Load(certPEM string, keyPEM string) (*CA, error) {
cert, err := tls.X509KeyPair([]byte(certPEM), []byte(keyPEM))
if err != nil {
return nil, fmt.Errorf("could not load CA: %w", err)
}

44
internal/certauthority/certauthority_test.go
View File

@ -12,6 +12,7 @@ import (
"crypto/x509/pkix"
"fmt"
"io"
"io/ioutil"
"strings"
"testing"
"time"
@ -19,6 +20,19 @@ import (
"github.com/stretchr/testify/require"
)
func loadFromFiles(t *testing.T, certPath string, keyPath string) (*CA, error) {
t.Helper()
certPEM, err := ioutil.ReadFile(certPath)
require.NoError(t, err)
keyPEM, err := ioutil.ReadFile(keyPath)
require.NoError(t, err)
ca, err := Load(string(certPEM), string(keyPEM))
return ca, err
}
func TestLoad(t *testing.T) {
tests := []struct {
name string
@ -26,30 +40,6 @@ func TestLoad(t *testing.T) {
keyPath string
wantErr string
}{
{
name: "missing cert",
certPath: "./testdata/cert-does-not-exist",
keyPath: "./testdata/test.key",
wantErr: "could not load CA: open ./testdata/cert-does-not-exist: no such file or directory",
},
{
name: "empty cert",
certPath: "./testdata/empty",
keyPath: "./testdata/test.key",
wantErr: "could not load CA: tls: failed to find any PEM data in certificate input",
},
{
name: "invalid cert",
certPath: "./testdata/invalid",
keyPath: "./testdata/test.key",
wantErr: "could not load CA: tls: failed to find any PEM data in certificate input",
},
{
name: "missing key",
certPath: "./testdata/test.crt",
keyPath: "./testdata/key-does-not-exist",
wantErr: "could not load CA: open ./testdata/key-does-not-exist: no such file or directory",
},
{
name: "empty key",
certPath: "./testdata/test.crt",
@ -83,7 +73,7 @@ func TestLoad(t *testing.T) {
for _, tt := range tests {
tt := tt
t.Run(tt.name, func(t *testing.T) {
ca, err := Load(tt.certPath, tt.keyPath)
ca, err := loadFromFiles(t, tt.certPath, tt.keyPath)
if tt.wantErr != "" {
require.EqualError(t, err, tt.wantErr)
return
@ -202,7 +192,7 @@ func (e *errSigner) Sign(_ io.Reader, _ []byte, _ crypto.SignerOpts) ([]byte, er
func TestIssue(t *testing.T) {
now := time.Date(2020, 7, 10, 12, 41, 12, 1234, time.UTC)
realCA, err := Load("./testdata/test.crt", "./testdata/test.key")
realCA, err := loadFromFiles(t, "./testdata/test.crt", "./testdata/test.key")
require.NoError(t, err)
tests := []struct {
@ -302,7 +292,7 @@ func TestIssue(t *testing.T) {
}
func TestIssuePEM(t *testing.T) {
realCA, err := Load("./testdata/test.crt", "./testdata/test.key")
realCA, err := loadFromFiles(t, "./testdata/test.crt", "./testdata/test.key")
require.NoError(t, err)
certPEM, keyPEM, err := realCA.IssuePEM(pkix.Name{CommonName: "Test Server"}, []string{"example.com"}, 10*time.Minute)

209
internal/certauthority/kubecertauthority/kubecertauthority.go Normal file
View File

@ -0,0 +1,209 @@
/*
Copyright 2020 VMware, Inc.
SPDX-License-Identifier: Apache-2.0
*/
// Package kubecertauthority implements a signer backed by the kubernetes controller-manager signing
// keys (accessed via the kubernetes Exec API).
package kubecertauthority
import (
"bytes"
"context"
"crypto/x509/pkix"
"fmt"
"sync"
"time"
"github.com/spf13/pflag"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/deprecated/scheme"
"k8s.io/client-go/kubernetes"
restclient "k8s.io/client-go/rest"
"k8s.io/client-go/tools/remotecommand"
"k8s.io/klog/v2"
"github.com/suzerain-io/placeholder-name/internal/certauthority"
"github.com/suzerain-io/placeholder-name/internal/constable"
)
// ErrNoKubeControllerManagerPod is returned when no kube-controller-manager pod is found on the cluster.
const ErrNoKubeControllerManagerPod = constable.Error("did not find kube-controller-manager pod")
const k8sAPIServerCACertPEMDefaultPath = "/etc/kubernetes/ca/ca.pem"
const k8sAPIServerCAKeyPEMDefaultPath = "/etc/kubernetes/ca/ca.key"
type signer interface {
IssuePEM(subject pkix.Name, dnsNames []string, ttl time.Duration) ([]byte, []byte, error)
}
type PodCommandExecutor interface {
Exec(podNamespace string, podName string, commandAndArgs ...string) (stdoutResult string, err error)
}
type kubeClientPodCommandExecutor struct {
kubeConfig *restclient.Config
kubeClient kubernetes.Interface
}
func NewPodCommandExecutor(kubeConfig *restclient.Config, kubeClient kubernetes.Interface) PodCommandExecutor {
return &kubeClientPodCommandExecutor{kubeConfig: kubeConfig, kubeClient: kubeClient}
}
func (s *kubeClientPodCommandExecutor) Exec(podNamespace string, podName string, commandAndArgs ...string) (string, error) {
request := s.kubeClient.
CoreV1().
RESTClient().
Post().
Namespace(podNamespace).
Resource("pods").
Name(podName).
SubResource("exec").
VersionedParams(&v1.PodExecOptions{
Stdin: false,
Stdout: true,
Stderr: false,
TTY: false,
Command: commandAndArgs,
}, scheme.ParameterCodec)
executor, err := remotecommand.NewSPDYExecutor(s.kubeConfig, "POST", request.URL())
if err != nil {
return "", err
}
var stdoutBuf bytes.Buffer
if err := executor.Stream(remotecommand.StreamOptions{Stdout: &stdoutBuf}); err != nil {
return "", err
}
return stdoutBuf.String(), nil
}
type CA struct {
kubeClient kubernetes.Interface
podCommandExecutor PodCommandExecutor
shutdown, done chan struct{}
lock sync.RWMutex
activeSigner signer
}
type ShutdownFunc func()
// New creates a new instance of a CA which is has loaded the kube API server's private key
// and is ready to issue certs, or an error. When successful, it also starts a goroutine
// to periodically reload the kube API server's private key in case it changed, and returns
// a function that can be used to shut down that goroutine.
func New(kubeClient kubernetes.Interface, podCommandExecutor PodCommandExecutor, tick <-chan time.Time) (*CA, ShutdownFunc, error) {
signer, err := createSignerWithAPIServerSecret(kubeClient, podCommandExecutor)
if err != nil {
// The initial load failed, so give up
return nil, nil, err
}
result := &CA{
kubeClient: kubeClient,
podCommandExecutor: podCommandExecutor,
activeSigner: signer,
shutdown: make(chan struct{}),
done: make(chan struct{}),
}
go result.refreshLoop(tick)
return result, result.shutdownRefresh, nil
}
func createSignerWithAPIServerSecret(kubeClient kubernetes.Interface, podCommandExecutor PodCommandExecutor) (signer, error) {
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
pod, err := findControllerManagerPod(ctx, kubeClient)
if err != nil {
return nil, err
}
certPath, keyPath := getKeypairFilePaths(pod)
certPEM, err := podCommandExecutor.Exec(pod.Namespace, pod.Name, "cat", certPath)
if err != nil {
return nil, err
}
keyPEM, err := podCommandExecutor.Exec(pod.Namespace, pod.Name, "cat", keyPath)
if err != nil {
return nil, err
}
return certauthority.Load(certPEM, keyPEM)
}
func (c *CA) refreshLoop(tick <-chan time.Time) {
for {
select {
case <-c.shutdown:
close(c.done)
return
case <-tick:
c.updateSigner()
}
}
}
func (c *CA) updateSigner() {
newSigner, err := createSignerWithAPIServerSecret(c.kubeClient, c.podCommandExecutor)
if err != nil {
klog.Errorf("could not create signer with API server secret: %s", err)
return
}
c.lock.Lock()
c.activeSigner = newSigner
c.lock.Unlock()
}
func (c *CA) shutdownRefresh() {
close(c.shutdown)
<-c.done
}
// IssuePEM issues a new server certificate for the given identity and duration, returning it as a pair of
// PEM-formatted byte slices for the certificate and private key.
func (c *CA) IssuePEM(subject pkix.Name, dnsNames []string, ttl time.Duration) ([]byte, []byte, error) {
c.lock.RLock()
signer := c.activeSigner
c.lock.RUnlock()
return signer.IssuePEM(subject, dnsNames, ttl)
}
func findControllerManagerPod(ctx context.Context, kubeClient kubernetes.Interface) (*v1.Pod, error) {
pods, err := kubeClient.CoreV1().Pods("kube-system").List(ctx, metav1.ListOptions{
LabelSelector: "component=kube-controller-manager",
FieldSelector: "status.phase=Running",
})
if err != nil {
return nil, fmt.Errorf("could not check for kube-controller-manager pod: %w", err)
}
for _, pod := range pods.Items {
return &pod, nil
}
return nil, ErrNoKubeControllerManagerPod
}
func getKeypairFilePaths(pod *v1.Pod) (string, string) {
certPath := getContainerArgByName(pod, "cluster-signing-cert-file", k8sAPIServerCACertPEMDefaultPath)
keyPath := getContainerArgByName(pod, "cluster-signing-key-file", k8sAPIServerCAKeyPEMDefaultPath)
return certPath, keyPath
}
func getContainerArgByName(pod *v1.Pod, name string, defaultValue string) string {
for _, container := range pod.Spec.Containers {
flagset := pflag.NewFlagSet("", pflag.ContinueOnError)
flagset.ParseErrorsWhitelist = pflag.ParseErrorsWhitelist{UnknownFlags: true}
var val string
flagset.StringVar(&val, name, "", "")
_ = flagset.Parse(append(container.Command, container.Args...))
if val != "" {
return val
}
}
return defaultValue
}

329
internal/certauthority/kubecertauthority/kubecertauthority_test.go Normal file
View File

@ -0,0 +1,329 @@
/*
Copyright 2020 VMware, Inc.
SPDX-License-Identifier: Apache-2.0
*/
package kubecertauthority
import (
"crypto/x509"
"crypto/x509/pkix"
"encoding/pem"
"errors"
"fmt"
"io/ioutil"
"testing"
"time"
"github.com/sclevine/spec"
"github.com/sclevine/spec/report"
"github.com/stretchr/testify/require"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
kubernetesfake "k8s.io/client-go/kubernetes/fake"
"k8s.io/klog/v2"
"github.com/suzerain-io/placeholder-name/internal/testutil"
)
type fakePodExecutor struct {
resultsToReturn []string
errorsToReturn []error
calledWithPodName []string
calledWithPodNamespace []string
calledWithCommandAndArgs [][]string
callCount int
}
func (s *fakePodExecutor) Exec(podNamespace string, podName string, commandAndArgs ...string) (string, error) {
s.calledWithPodNamespace = append(s.calledWithPodNamespace, podNamespace)
s.calledWithPodName = append(s.calledWithPodName, podName)
s.calledWithCommandAndArgs = append(s.calledWithCommandAndArgs, commandAndArgs)
result := s.resultsToReturn[s.callCount]
var err error = nil
if s.errorsToReturn != nil {
err = s.errorsToReturn[s.callCount]
}
s.callCount++
if err != nil {
return "", err
}
return result, nil
}
func TestCA(t *testing.T) {
spec.Run(t, "CA", func(t *testing.T, when spec.G, it spec.S) {
var r *require.Assertions
var fakeCertPEM, fakeKeyPEM string
var fakeCert2PEM, fakeKey2PEM string
var fakePod *corev1.Pod
var kubeAPIClient *kubernetesfake.Clientset
var fakeExecutor *fakePodExecutor
var neverTicker <-chan time.Time
var logger *testutil.TranscriptLogger
it.Before(func() {
r = require.New(t)
loadFile := func(filename string) string {
bytes, err := ioutil.ReadFile(filename)
r.NoError(err)
return string(bytes)
}
fakeCertPEM = loadFile("./testdata/test.crt")
fakeKeyPEM = loadFile("./testdata/test.key")
fakeCert2PEM = loadFile("./testdata/test2.crt")
fakeKey2PEM = loadFile("./testdata/test2.key")
fakePod = &corev1.Pod{
TypeMeta: metav1.TypeMeta{},
ObjectMeta: metav1.ObjectMeta{
Name: "fake-pod",
Namespace: "kube-system",
Labels: map[string]string{"component": "kube-controller-manager"},
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{{Name: "kube-controller-manager"}},
},
Status: corev1.PodStatus{
Phase: "Running",
},
}
kubeAPIClient = kubernetesfake.NewSimpleClientset()
fakeExecutor = &fakePodExecutor{
resultsToReturn: []string{
fakeCertPEM,
fakeKeyPEM,
fakeCert2PEM,
fakeKey2PEM,
},
}
logger = testutil.NewTranscriptLogger(t)
klog.SetLogger(logger) // this is unfortunately a global logger, so can't run these tests in parallel :(
})
it.After(func() {
klog.SetLogger(nil)
})
when("the kube-controller-manager pod is found with default CLI flag values", func() {
it.Before(func() {
err := kubeAPIClient.Tracker().Add(fakePod)
r.NoError(err)
})
when("the exec commands return the API server's keypair", func() {
it("finds the API server's signing key and uses it to issue certificates", func() {
fakeTicker := make(chan time.Time)
subject, shutdownFunc, err := New(kubeAPIClient, fakeExecutor, fakeTicker)
r.NoError(err)
r.NotNil(shutdownFunc)
defer shutdownFunc()
r.Equal(2, fakeExecutor.callCount)
r.Equal("kube-system", fakeExecutor.calledWithPodNamespace[0])
r.Equal("fake-pod", fakeExecutor.calledWithPodName[0])
r.Equal([]string{"cat", "/etc/kubernetes/ca/ca.pem"}, fakeExecutor.calledWithCommandAndArgs[0])
r.Equal("kube-system", fakeExecutor.calledWithPodNamespace[1])
r.Equal("fake-pod", fakeExecutor.calledWithPodName[1])
r.Equal([]string{"cat", "/etc/kubernetes/ca/ca.key"}, fakeExecutor.calledWithCommandAndArgs[1])
// Validate that we can issue a certificate signed by the original API server CA.
certPEM, keyPEM, err := subject.IssuePEM(
pkix.Name{CommonName: "Test Server"},
[]string{"example.com"},
10*time.Minute,
)
r.NoError(err)
validCert := testutil.ValidateCertificate(t, fakeCertPEM, string(certPEM))
validCert.RequireDNSName("example.com")
validCert.RequireLifetime(time.Now(), time.Now().Add(10*time.Minute), 2*time.Minute)
validCert.RequireMatchesPrivateKey(string(keyPEM))
// Tick the timer and wait for another refresh loop to complete.
fakeTicker <- time.Now()
var secondCertPEM, secondKeyPEM string
r.Eventually(func() bool {
certPEM, keyPEM, err := subject.IssuePEM(
pkix.Name{CommonName: "Test Server"},
[]string{"example.com"},
10*time.Minute,
)
r.NoError(err)
secondCertPEM = string(certPEM)
secondKeyPEM = string(keyPEM)
block, _ := pem.Decode(certPEM)
require.NotNil(t, block)
parsed, err := x509.ParseCertificate(block.Bytes)
require.NoError(t, err)
// Validate the created cert using the second API server CA.
roots := x509.NewCertPool()
require.True(t, roots.AppendCertsFromPEM([]byte(fakeCert2PEM)))
opts := x509.VerifyOptions{Roots: roots}
_, err = parsed.Verify(opts)
return err == nil
}, 5*time.Second, 100*time.Millisecond)
validCert2 := testutil.ValidateCertificate(t, fakeCert2PEM, secondCertPEM)
validCert2.RequireDNSName("example.com")
validCert2.RequireLifetime(time.Now(), time.Now().Add(10*time.Minute), 2*time.Minute)
validCert2.RequireMatchesPrivateKey(secondKeyPEM)
})
})
when("the exec commands return the API server's keypair the first time but subsequently fails", func() {
it.Before(func() {
fakeExecutor.errorsToReturn = []error{nil, nil, fmt.Errorf("some exec error")}
})
it("logs an error message", func() {
fakeTicker := make(chan time.Time)
subject, shutdownFunc, err := New(kubeAPIClient, fakeExecutor, fakeTicker)
r.NoError(err)
r.NotNil(shutdownFunc)
defer shutdownFunc()
r.Equal(2, fakeExecutor.callCount)
// Tick the timer and wait for another refresh loop to complete.
fakeTicker <- time.Now()
// Wait for there to be a log output and require that it matches our expectation.
r.Eventually(func() bool { return len(logger.Transcript()) >= 1 }, 5*time.Second, 10*time.Millisecond)
r.Contains(logger.Transcript()[0].Message, "could not create signer with API server secret: some exec error")
r.Equal(logger.Transcript()[0].Level, "error")
// Validate that we can still issue a certificate signed by the original API server CA.
certPEM, _, err := subject.IssuePEM(
pkix.Name{CommonName: "Test Server"},
[]string{"example.com"},
10*time.Minute,
)
r.NoError(err)
testutil.ValidateCertificate(t, fakeCertPEM, string(certPEM))
})
})
when("the exec commands succeed but return garbage", func() {
it.Before(func() {
fakeExecutor.resultsToReturn = []string{"not a cert", "not a private key"}
})
it("returns an error", func() {
subject, shutdownFunc, err := New(kubeAPIClient, fakeExecutor, neverTicker)
r.Nil(subject)
r.Nil(shutdownFunc)
r.EqualError(err, "could not load CA: tls: failed to find any PEM data in certificate input")
})
})
when("the first exec command returns an error", func() {
it.Before(func() {
fakeExecutor.errorsToReturn = []error{fmt.Errorf("some error"), nil}
})
it("returns an error", func() {
subject, shutdownFunc, err := New(kubeAPIClient, fakeExecutor, neverTicker)
r.Nil(subject)
r.Nil(shutdownFunc)
r.EqualError(err, "some error")
})
})
when("the second exec command returns an error", func() {
it.Before(func() {
fakeExecutor.errorsToReturn = []error{nil, fmt.Errorf("some error")}
})
it("returns an error", func() {
subject, shutdownFunc, err := New(kubeAPIClient, fakeExecutor, neverTicker)
r.Nil(subject)
r.Nil(shutdownFunc)
r.EqualError(err, "some error")
})
})
})
when("the kube-controller-manager pod is found with non-default CLI flag values", func() {
it.Before(func() {
fakePod.Spec.Containers[0].Command = []string{
"kube-controller-manager",
"--cluster-signing-cert-file=/etc/kubernetes/ca/non-default.pem",
}
fakePod.Spec.Containers[0].Args = []string{
"--cluster-signing-key-file=/etc/kubernetes/ca/non-default.key",
}
err := kubeAPIClient.Tracker().Add(fakePod)
r.NoError(err)
})
it("finds the API server's signing key and uses it to issue certificates", func() {
_, shutdownFunc, err := New(kubeAPIClient, fakeExecutor, neverTicker)
r.NoError(err)
r.NotNil(shutdownFunc)
defer shutdownFunc()
r.Equal(2, fakeExecutor.callCount)
r.Equal("kube-system", fakeExecutor.calledWithPodNamespace[0])
r.Equal("fake-pod", fakeExecutor.calledWithPodName[0])
r.Equal([]string{"cat", "/etc/kubernetes/ca/non-default.pem"}, fakeExecutor.calledWithCommandAndArgs[0])
r.Equal("kube-system", fakeExecutor.calledWithPodNamespace[1])
r.Equal("fake-pod", fakeExecutor.calledWithPodName[1])
r.Equal([]string{"cat", "/etc/kubernetes/ca/non-default.key"}, fakeExecutor.calledWithCommandAndArgs[1])
})
})
when("the kube-controller-manager pod is found with non-default CLI flag values separated by spaces", func() {
it.Before(func() {
fakePod.Spec.Containers[0].Command = []string{
"kube-controller-manager",
"--cluster-signing-cert-file", "/etc/kubernetes/ca/non-default.pem",
"--cluster-signing-key-file", "/etc/kubernetes/ca/non-default.key",
"--foo=bar",
}
err := kubeAPIClient.Tracker().Add(fakePod)
r.NoError(err)
})
it("finds the API server's signing key and uses it to issue certificates", func() {
_, shutdownFunc, err := New(kubeAPIClient, fakeExecutor, neverTicker)
r.NoError(err)
r.NotNil(shutdownFunc)
defer shutdownFunc()
r.Equal(2, fakeExecutor.callCount)
r.Equal("kube-system", fakeExecutor.calledWithPodNamespace[0])
r.Equal("fake-pod", fakeExecutor.calledWithPodName[0])
r.Equal([]string{"cat", "/etc/kubernetes/ca/non-default.pem"}, fakeExecutor.calledWithCommandAndArgs[0])
r.Equal("kube-system", fakeExecutor.calledWithPodNamespace[1])
r.Equal("fake-pod", fakeExecutor.calledWithPodName[1])
r.Equal([]string{"cat", "/etc/kubernetes/ca/non-default.key"}, fakeExecutor.calledWithCommandAndArgs[1])
})
})
when("the kube-controller-manager pod is not found", func() {
it("returns an error", func() {
subject, shutdownFunc, err := New(kubeAPIClient, fakeExecutor, neverTicker)
r.Nil(subject)
r.Nil(shutdownFunc)
r.True(errors.Is(err, ErrNoKubeControllerManagerPod))
})
})
}, spec.Report(report.Terminal{}))
}

17
internal/certauthority/kubecertauthority/testdata/test.crt vendored Normal file
View File

@ -0,0 +1,17 @@
-----BEGIN CERTIFICATE-----
MIICyDCCAbCgAwIBAgIBADANBgkqhkiG9w0BAQsFADAVMRMwEQYDVQQDEwprdWJl
cm5ldGVzMB4XDTIwMDcyNTIxMDQxOFoXDTMwMDcyMzIxMDQxOFowFTETMBEGA1UE
AxMKa3ViZXJuZXRlczCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAL3K
hYv2gIQ1Dwzh2cWMid+ofAnvLIfV2Xv61vTLGprUI+XUqB4/gtf6X6UNn0Lett2n
d8p4wy7hw73hU/ggdvmWJvqBrSjc3JGfy+kj66fKXX+PTlbL7QbwiRvcSqIXIWlV
lHHxECWrED8jCulw/NVqfook/h5iNUCT9yswSJr/0fImiVnoTlIoEYG2eCNejZ5c
g39uD3ZTqd9ZxWwSLLnI+2kpJnZBPcd1ZQ8AQqzDgZtYRCqacn5gckQUKZWKQlxo
Eft6g1XHJouAWAZw7hEtk0v8rG0/eKF7wamxFi6BFVlbjWBsB4T9rApbdBWTKeCJ
Hv8fv5RMFSzpT3uzTO8CAwEAAaMjMCEwDgYDVR0PAQH/BAQDAgKkMA8GA1UdEwEB
/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBACh5RhbxqJe+Z/gc17cZhKNmdiwu
I2pLp3QBfwvN+Wbmajzw/7rYhY0d8JYVTJzXSCPWi6UAKxAtXOLF8WIIf9i39n6R
uKOBGW14FzzGyRJiD3qaG/JTvEW+SLhwl68Ndr5LHSnbugAqq31abcQy6Zl9v5A8
JKC97Lj/Sn8rj7opKy4W3oq7NCQsAb0zh4IllRF6UvSnJySfsg7xdXHHpxYDHtOS
XcOu5ySUIZTgFe9RfeUZlGZ5xn0ckMlQ7qW2Wx1q0OVWw5us4NtkGqKrHG4Tn1X7
uwo/Yytn5sDxrDv1/oii6AZOCsTPre4oD3wz4nmVzCVJcgrqH4Q24hT8WNg=
-----END CERTIFICATE-----

27
internal/certauthority/kubecertauthority/testdata/test.key vendored Normal file
View File

@ -0,0 +1,27 @@
-----BEGIN RSA PRIVATE KEY-----
MIIEogIBAAKCAQEAvcqFi/aAhDUPDOHZxYyJ36h8Ce8sh9XZe/rW9MsamtQj5dSo
Hj+C1/pfpQ2fQt623ad3ynjDLuHDveFT+CB2+ZYm+oGtKNzckZ/L6SPrp8pdf49O
VsvtBvCJG9xKohchaVWUcfEQJasQPyMK6XD81Wp+iiT+HmI1QJP3KzBImv/R8iaJ
WehOUigRgbZ4I16NnlyDf24PdlOp31nFbBIsucj7aSkmdkE9x3VlDwBCrMOBm1hE
KppyfmByRBQplYpCXGgR+3qDVccmi4BYBnDuES2TS/ysbT94oXvBqbEWLoEVWVuN
YGwHhP2sClt0FZMp4Ike/x+/lEwVLOlPe7NM7wIDAQABAoIBAFC1tUEmHNUcM0BJ
M3D9KQzB+63F1mwVlx1QOOV1EeVR3co5Ox1R6PSr9sycFGQ9jgqI0zp5TJe9Tp6L
GkhklfPh1MWnK9o6wlnzWKXWrrp2Jni+mpPyuOPAmq4Maniv2XeP+0bROwqpyojv
AA7yC7M+TH226ZJGNVs3EV9+cwHml0yuzBfIJn/rv/w2g+WRKM/MC0S7k2d8bRlA
NycKVGAGBhKTltjoVYOeh6aHEpSjK8zfaePjo5dYJvoVIli60YCgcJOU/8jXT+Np
1Fm7tRvAtj3pUp0Sqdaf2RUzh9jfJp2VFCHuSJ6TPqArOyQojtMcTHF0TiW7xrHP
xOCRIAECgYEAwGBPU7vdthMJBg+ORUoGQQaItTeJvQwIqJvbKD2osp4jhS1dGZBw
W30GKEc/gd8JNtOq9BBnMicPF7hktuy+bSPv41XPud67rSSO7Tsw20C10gFRq06B
zIJWFAUqK3IkvVc3VDmtSLSDox4QZ/BdqaMlQ5y5JCsC5kThmkZFlO8CgYEA/I9X
YHi6RioMJE1fqOHJL4DDjlezmcuRrD7fE5InKbtJZ2JhGYOX/C0KXnHTOWTCDxxN
FBvpvD6Xv5o3PhB9Z6k2fqvJ4GS8urkG/KU4xcC+bak+9ava8oaiSqG16zD9NH2P
jJ60NrbLl1J0pU9fiwuFVUKJ4hDZOfN9RqYdyAECgYAVwo8WhJiGgM6zfcz073OX
pVqPTPHqjVLpZ3+5pIfRdGvGI6R1QM5EuvaYVb7MPOM47WZX5wcVOC/P2g6iVlMP
21HGIC2384a9BfaYxOo40q/+SiHnw6CQ9mkwKIllkqqvNA9RGpkMMUb2i28For2l
c4vCgxa6DZdtXns6TRqPxwKBgCfY5cxOv/T6BVhk7MbUeM2J31DB/ZAyUhV/Bess
kAlBh19MYk2IOZ6L7KriApV3lDaWHIMjtEkDByYvyq98Io0MYZCywfMpca10K+oI
l2B7/I+IuGpCZxUEsO5dfTpSTGDPvqpND9niFVUWqVi7oTNq6ep9yQtl5SADjqxq
4SABAoGAIm0hUg1wtcS46cGLy6PIkPM5tocTSghtz4vFsuk/i4QA9GBoBO2gH6ty
+kJHmeaXt2dmgySp0QAWit5UlceEumB0NXnAdJZQxeGSFSyYkDWhwXd8wDceKo/1
LfCU6Dk8IN/SsppVUWXQ2rlORvxlrHeCio8o0kS9Yiu55WMYg4g=
-----END RSA PRIVATE KEY-----

17
internal/certauthority/kubecertauthority/testdata/test2.crt vendored Normal file
View File

@ -0,0 +1,17 @@
-----BEGIN CERTIFICATE-----
MIICyDCCAbCgAwIBAgIBADANBgkqhkiG9w0BAQsFADAVMRMwEQYDVQQDEwprdWJl
cm5ldGVzMB4XDTIwMDgxODE2NDEzNloXDTMwMDgxNjE2NDEzNlowFTETMBEGA1UE
AxMKa3ViZXJuZXRlczCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALH7
C2JpttDi3mxpD4bd+BZucCrS8XF2YwqYAr42HePp++PBnlUFqWmtPc9/bmo+7+7z
iAAlnAV0pJWP+HR/PskX8MRcFAA1HoXLa37Q4SuBBQG+JE+AeaOObmQYaCFv55ej
UF4+JIoQOdlbYEMYSI07el0cxQL4Io/CHJ3p7AtNElxjDuMK4B9W8NiCse3p7Uf+
Qje4we1TYOfcpAM0jpBPHK9vCBCpX+j52S5DUTRVIk9kye3lCDmWOXH/fhj/aJTM
1MP/hThbl2wIbFuv1bpa0kXNZs8xB63dtqROQ+lCghDmuayRmzwXl2PX6IgFFcjV
yAgjXrZqjihs+mY8eT0CAwEAAaMjMCEwDgYDVR0PAQH/BAQDAgKkMA8GA1UdEwEB
/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAE+Saqk2EyuIx1rxFWrOwpTi5q/B
p/TwEtrmrFIRVPnGeBnhyfbGXPDMkzIY1mEvztu8H+pm5RPyhQYLsuwzYiYMQyxX
yL9VvO7uydn7+3zX7oknQ5qAvN3nmItNyOKw3MRIKGsySNuTQ5JPtU/ufGlEivbK
vNaDBqjKrBvwhIKMdV9/xYSyeBhSSWr/6W1tAk+XbHhQH1M78rdwGN5SI75L4FGu
13kn/W2n8pE17TAY88B1YGKhsLSvf8KrFNYv+UUmzh2WstECKSlnbrSM+boMlGJn
XahE8M23fieB+SaenQdOezrY4GAnXQ3qToDlhdYAOkWhcGDct47VRM93whY=
-----END CERTIFICATE-----

27
internal/certauthority/kubecertauthority/testdata/test2.key vendored Normal file
View File

@ -0,0 +1,27 @@
-----BEGIN RSA PRIVATE KEY-----
MIIEogIBAAKCAQEAsfsLYmm20OLebGkPht34Fm5wKtLxcXZjCpgCvjYd4+n748Ge
VQWpaa09z39uaj7v7vOIACWcBXSklY/4dH8+yRfwxFwUADUehctrftDhK4EFAb4k
T4B5o45uZBhoIW/nl6NQXj4kihA52VtgQxhIjTt6XRzFAvgij8IcnensC00SXGMO
4wrgH1bw2IKx7entR/5CN7jB7VNg59ykAzSOkE8cr28IEKlf6PnZLkNRNFUiT2TJ
7eUIOZY5cf9+GP9olMzUw/+FOFuXbAhsW6/VulrSRc1mzzEHrd22pE5D6UKCEOa5
rJGbPBeXY9foiAUVyNXICCNetmqOKGz6Zjx5PQIDAQABAoIBAD06klYO7De8dKxz
EEZjgn+lCq2Q2EMiaTwxw2/QikPoMSHPcDrrsbaLROJngoLGmCBqY3U5ew1dbWmO
l/jr9ZuUwt2ql67il1eL/bUpAu3GewR4d2FqX25nB48j3l7ycof2RSXG1ycwIdam
2tz6M6tytMvno9c7qhguvU2ONghEreXG3YYLdf9l97aB+p6GdXhwty22b7tAVwp1
GKn79kVYgmL86lph9hBPqtHuG1LHZUiFodr2iWXSu3H/265OD58a33ZO3iyfFI0s
PPy87ZN0r+1hGpoKKkDe63udOYgAG6xmIea/1Pdn9Eg87tueoeC7XcUpdaCJlKaF
tqCusEECgYEA60rWyXxTFKJ4QdVaqXoWMA4cQkT73RxznSKwkN/Svk8TVv+p5s5Y
oYKN4qyMzxvQzu+QNWpd1yTveCmmEynz457ELpGtidtiJdm7xZMdMGrU02eCL9mZ
ERbtfAkbEAKvN8D73fWyzghKv4dgcQptmsqZlYYc4vpwHveK+/N5lukCgYEAwaT3
iMTWCv7Vp87iKrzNUAH4iBWlazwbE+EDEnHVw26Y82fhgEgxiU2UvFSaIVhGpaCz
MYSXSdRcQTHgCoJLPfWHUHTJPqf36KfAJfdaxxjzTTbNYjUOkdcUD1bcNrm0yjoY
nR4zK1FPw86ODMYtBpfkyL7ZX8G1v5pRL/6/gzUCgYBzgwQ7Wmu3H6QGPeYKecNW
yDabWh6ECKnBpPwlw5xEjbGi7lTM2NSuRde+RpPCQZebYATeFGAJdTqTNW8wzVHM
l28cpawal7dxeZkzf+u+j1P4jUJel2cL+sOQNzAwBgFbT8TWzP6BI5T+vklcdZAl
g/0uaO7Zh7Vvnnt/AaLZsQKBgGfbHzuGPjoFdPecOKatPfxUIkRyP5bk1KzzuF8T
GI/JaFTbeREBJzg5mLTtNwD9RF6ecpzzPOTG9Xet1Tgtq0cewSUAjdKB6a8pESAL
qu8vTYYzBzJNvHOxg7u6XT8omHMBd6QEx3LLGFmvFXZ6bzmjC3wzB4iY7u5FSJfS
LEqlAoGAb0rbJ85vrJopbx8lzhJjyaJfM8+A3oQg1K3TphHrcgkUc8qx8QEosziM
wzYKSBlXd2bxMibyd0mTEMNl4/BqofaKoqof9gBIbkamwXOO8s7IgKxQAfr1R/z8
tHBW/g0QWPB+qtaVDtHwyQLlxjx8HD7atIo8d/do9ruwVaf+r6g=
-----END RSA PRIVATE KEY-----

30
internal/controller/apicerts/certs_manager_test.go
View File

@ -7,9 +7,6 @@ package apicerts
import (
"context"
"crypto/tls"
"crypto/x509"
"encoding/pem"
"errors"
"testing"
"time"
@ -222,29 +219,10 @@ func TestManagerControllerSync(t *testing.T) {
r.NotEmpty(actualCertChain)
// Validate the created cert using the CA, and also validate the cert's hostname
roots := x509.NewCertPool()
ok := roots.AppendCertsFromPEM([]byte(actualCACert))
r.True(ok)
block, _ := pem.Decode([]byte(actualCertChain))
r.NotNil(block)
parsedCert, err := x509.ParseCertificate(block.Bytes)
r.NoError(err)
serviceEndpoint := "placeholder-name-api." + installedInNamespace + ".svc"
opts := x509.VerifyOptions{
DNSName: serviceEndpoint,
Roots: roots,
}
_, err = parsedCert.Verify(opts)
r.NoError(err)
r.Contains(parsedCert.DNSNames, serviceEndpoint, "expected an explicit DNS SAN, not just Common Name")
// Check the created cert's validity bounds
r.WithinDuration(time.Now(), parsedCert.NotBefore, time.Minute*2)
r.WithinDuration(time.Now().Add(24*365*time.Hour), parsedCert.NotAfter, time.Minute*2)
// Check that the private key and cert chain match
_, err = tls.X509KeyPair([]byte(actualCertChain), []byte(actualPrivateKey))
r.NoError(err)
validCert := testutil.ValidateCertificate(t, actualCACert, actualCertChain)
validCert.RequireDNSName("placeholder-name-api." + installedInNamespace + ".svc")
validCert.RequireLifetime(time.Now(), time.Now().Add(24*365*time.Hour), 2*time.Minute)
validCert.RequireMatchesPrivateKey(actualPrivateKey)
// Make sure we updated the APIService caBundle and left it otherwise unchanged
r.Len(aggregatorAPIClient.Actions(), 2)

2
internal/controllermanager/prepare_controllers.go
View File

@ -97,7 +97,7 @@ func createClients() (
placeholderClient *placeholderclientset.Clientset,
err error,
) {
// Load the Kubernetes client configuration (kubeconfig),
// Load the Kubernetes client configuration.
kubeConfig, err := restclient.InClusterConfig()
if err != nil {
return nil, nil, nil, fmt.Errorf("could not load in-cluster configuration: %w", err)

7
internal/registry/credentialrequest/rest_test.go
View File

@ -425,9 +425,10 @@ func TestCreate(t *testing.T) {
}
func requireOneLogStatement(r *require.Assertions, logger *testutil.TranscriptLogger, messageContains string) {
r.Len(logger.Transcript, 1)
r.Equal("info", logger.Transcript[0].Level)
r.Contains(logger.Transcript[0].Message, messageContains)
transcript := logger.Transcript()
r.Len(transcript, 1)
r.Equal("info", transcript[0].Level)
r.Contains(transcript[0].Message, messageContains)
}
func callCreate(ctx context.Context, storage *REST, credentialRequest *placeholderapi.CredentialRequest) (runtime.Object, error) {

62
internal/server/server.go
View File

@ -10,17 +10,21 @@ import (
"context"
"fmt"
"io"
"time"
"github.com/spf13/cobra"
genericapiserver "k8s.io/apiserver/pkg/server"
genericoptions "k8s.io/apiserver/pkg/server/options"
"k8s.io/apiserver/plugin/pkg/authenticator/token/webhook"
"k8s.io/client-go/kubernetes"
restclient "k8s.io/client-go/rest"
"github.com/suzerain-io/placeholder-name/internal/apiserver"
"github.com/suzerain-io/placeholder-name/internal/certauthority"
"github.com/suzerain-io/placeholder-name/internal/certauthority/kubecertauthority"
"github.com/suzerain-io/placeholder-name/internal/controllermanager"
"github.com/suzerain-io/placeholder-name/internal/downward"
"github.com/suzerain-io/placeholder-name/internal/provider"
"github.com/suzerain-io/placeholder-name/internal/registry/credentialrequest"
placeholderv1alpha1 "github.com/suzerain-io/placeholder-name/kubernetes/1.19/api/apis/placeholder/v1alpha1"
"github.com/suzerain-io/placeholder-name/pkg/config"
)
@ -30,10 +34,8 @@ type App struct {
cmd *cobra.Command
// CLI flags
configPath string
downwardAPIPath string
clusterSigningCertFilePath string
clusterSigningKeyFilePath string
configPath string
downwardAPIPath string
}
// This is ignored for now because we turn off etcd storage below, but this is
@ -87,20 +89,6 @@ func addCommandlineFlagsToCommand(cmd *cobra.Command, app *App) {
"/etc/podinfo",
"path to Downward API volume mount",
)
cmd.Flags().StringVar(
&app.clusterSigningCertFilePath,
"cluster-signing-cert-file",
"",
"path to cluster signing certificate",
)
cmd.Flags().StringVar(
&app.clusterSigningKeyFilePath,
"cluster-signing-key-file",
"",
"path to cluster signing private key",
)
}
// Boot the aggregated API server, which will in turn boot the controllers.
@ -112,10 +100,11 @@ func (a *App) runServer(ctx context.Context) error {
}
// Load the Kubernetes cluster signing CA.
k8sClusterCA, err := certauthority.Load(a.clusterSigningCertFilePath, a.clusterSigningKeyFilePath)
k8sClusterCA, shutdownCA, err := getClusterCASigner()
if err != nil {
return fmt.Errorf("could not load cluster signing CA: %w", err)
return err
}
defer shutdownCA()
// Create a WebhookTokenAuthenticator.
webhookTokenAuthenticator, err := config.NewWebhook(cfg.WebhookConfig)
@ -169,11 +158,40 @@ func (a *App) runServer(ctx context.Context) error {
return server.GenericAPIServer.PrepareRun().Run(ctx.Done())
}
func getClusterCASigner() (*kubecertauthority.CA, kubecertauthority.ShutdownFunc, error) {
// Load the Kubernetes client configuration.
kubeConfig, err := restclient.InClusterConfig()
if err != nil {
return nil, nil, fmt.Errorf("could not load in-cluster configuration: %w", err)
}
// Connect to the core Kubernetes API.
kubeClient, err := kubernetes.NewForConfig(kubeConfig)
if err != nil {
return nil, nil, fmt.Errorf("could not initialize Kubernetes client: %w", err)
}
// Make a clock tick that triggers a periodic refresh.
ticker := time.NewTicker(5 * time.Minute)
// Make a CA which uses the Kubernetes cluster API server's signing certs.
k8sClusterCA, shutdownCA, err := kubecertauthority.New(
kubeClient,
kubecertauthority.NewPodCommandExecutor(kubeConfig, kubeClient),
ticker.C,
)
if err != nil {
ticker.Stop()
return nil, nil, fmt.Errorf("could not load cluster signing CA: %w", err)
}
return k8sClusterCA, func() { shutdownCA(); ticker.Stop() }, nil
}
// Create a configuration for the aggregated API server.
func getAggregatedAPIServerConfig(
dynamicCertProvider provider.DynamicTLSServingCertProvider,
webhookTokenAuthenticator *webhook.WebhookTokenAuthenticator,
ca *certauthority.CA,
ca credentialrequest.CertIssuer,
startControllersPostStartHook func(context.Context),
) (*apiserver.Config, error) {
recommendedOptions := genericoptions.NewRecommendedOptions(

10
internal/server/server_test.go
View File

@ -25,12 +25,10 @@ Usage:
placeholder-name-server [flags]
Flags:
--cluster-signing-cert-file string path to cluster signing certificate
--cluster-signing-key-file string path to cluster signing private key
-c, --config string path to configuration file (default "placeholder-name.yaml")
--downward-api-path string path to Downward API volume mount (default "/etc/podinfo")
-h, --help help for placeholder-name-server
--log-flush-frequency duration Maximum number of seconds between log flushes (default 5s)
-c, --config string path to configuration file (default "placeholder-name.yaml")
--downward-api-path string path to Downward API volume mount (default "/etc/podinfo")
-h, --help help for placeholder-name-server
--log-flush-frequency duration Maximum number of seconds between log flushes (default 5s)
`
func TestCommand(t *testing.T) {

73
internal/testutil/certs.go Normal file
View File

@ -0,0 +1,73 @@
/*
Copyright 2020 VMware, Inc.
SPDX-License-Identifier: Apache-2.0
*/
package testutil
import (
"crypto/tls"
"crypto/x509"
"encoding/pem"
"testing"
"time"
"github.com/stretchr/testify/require"
)
type ValidCert struct {
t *testing.T
roots *x509.CertPool
certPEM string
parsed *x509.Certificate
}
// ValidateCertificate validates a certificate and provides an object for asserting properties of the certificate.
func ValidateCertificate(t *testing.T, caPEM string, certPEM string) *ValidCert {
t.Helper()
block, _ := pem.Decode([]byte(certPEM))
require.NotNil(t, block)
parsed, err := x509.ParseCertificate(block.Bytes)
require.NoError(t, err)
// Validate the created cert using the CA.
roots := x509.NewCertPool()
require.True(t, roots.AppendCertsFromPEM([]byte(caPEM)))
opts := x509.VerifyOptions{Roots: roots}
_, err = parsed.Verify(opts)
require.NoError(t, err)
return &ValidCert{
t: t,
roots: roots,
certPEM: certPEM,
parsed: parsed,
}
}
// RequireDNSName asserts that the certificate matches the provided DNS name.
func (v *ValidCert) RequireDNSName(expectDNSName string) {
v.t.Helper()
opts := x509.VerifyOptions{
Roots: v.roots,
DNSName: expectDNSName,
}
_, err := v.parsed.Verify(opts)
require.NoError(v.t, err)
require.Contains(v.t, v.parsed.DNSNames, expectDNSName, "expected an explicit DNS SAN, not just Common Name")
}
// RequireLifetime asserts that the lifetime of the certificate matches the expected timestamps.
func (v *ValidCert) RequireLifetime(expectNotBefore time.Time, expectNotAfter time.Time, delta time.Duration) {
v.t.Helper()
require.WithinDuration(v.t, expectNotBefore, v.parsed.NotBefore, delta)
require.WithinDuration(v.t, expectNotAfter, v.parsed.NotAfter, delta)
}
// RequireMatchesPrivateKey asserts that the public key in the certificate matches the provided private key.
func (v *ValidCert) RequireMatchesPrivateKey(keyPEM string) {
v.t.Helper()
_, err := tls.X509KeyPair([]byte(v.certPEM), []byte(keyPEM))
require.NoError(v.t, err)
}

20
internal/testutil/transcript_logger.go
View File

@ -7,6 +7,7 @@ package testutil
import (
"fmt"
"sync"
"testing"
"github.com/go-logr/logr"
@ -14,7 +15,8 @@ import (
type TranscriptLogger struct {
t *testing.T
Transcript []TranscriptLogMessage
lock sync.Mutex
transcript []TranscriptLogMessage
}
var _ logr.Logger = &TranscriptLogger{}
@ -28,15 +30,27 @@ func NewTranscriptLogger(t *testing.T) *TranscriptLogger {
return &TranscriptLogger{t: t}
}
func (log *TranscriptLogger) Transcript() []TranscriptLogMessage {
log.lock.Lock()
defer log.lock.Unlock()
result := make([]TranscriptLogMessage, 0, len(log.transcript))
result = append(result, log.transcript...)
return result
}
func (log *TranscriptLogger) Info(msg string, keysAndValues ...interface{}) {
log.Transcript = append(log.Transcript, TranscriptLogMessage{
log.lock.Lock()
defer log.lock.Unlock()
log.transcript = append(log.transcript, TranscriptLogMessage{
Level: "info",
Message: fmt.Sprintf(msg, keysAndValues...),
})
}
func (log *TranscriptLogger) Error(err error, msg string, keysAndValues ...interface{}) {
log.Transcript = append(log.Transcript, TranscriptLogMessage{
log.lock.Lock()
defer log.lock.Unlock()
log.transcript = append(log.transcript, TranscriptLogMessage{
Level: "error",
Message: fmt.Sprintf("%s: %v -- %v", msg, err, keysAndValues),
})

24
test/integration/app_availability_test.go
View File

@ -11,23 +11,39 @@ import (
"time"
"github.com/stretchr/testify/require"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/suzerain-io/placeholder-name/test/library"
)
func TestAppAvailability(t *testing.T) {
func TestGetDeployment(t *testing.T) {
library.SkipUnlessIntegration(t)
namespaceName := library.Getenv(t, "PLACEHOLDER_NAME_NAMESPACE")
daemonSetName := library.Getenv(t, "PLACEHOLDER_NAME_APP_NAME")
deploymentName := library.Getenv(t, "PLACEHOLDER_NAME_APP_NAME")
client := library.NewClientset(t)
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
daemonSet, err := client.AppsV1().DaemonSets(namespaceName).Get(ctx, daemonSetName, metav1.GetOptions{})
appDeployment, err := client.AppsV1().Deployments(namespaceName).Get(ctx, deploymentName, metav1.GetOptions{})
require.NoError(t, err)
require.GreaterOrEqual(t, daemonSet.Status.NumberAvailable, int32(1))
cond := getDeploymentCondition(appDeployment.Status, appsv1.DeploymentAvailable)
require.NotNil(t, cond)
require.Equalf(t, corev1.ConditionTrue, cond.Status, "app should be available: %s", library.Sdump(appDeployment))
}
// getDeploymentCondition returns the condition with the provided type.
// Copied from k8s.io/kubectl/pkg/util/deployment/deployment.go to prevent us from vendoring the world.
func getDeploymentCondition(status appsv1.DeploymentStatus, condType appsv1.DeploymentConditionType) *appsv1.DeploymentCondition {
for i := range status.Conditions {
c := status.Conditions[i]
if c.Type == condType {
return &c
}
}
return nil
}