Switch back to an exec-based approach to grab the controller-manager CA. (#65)
This switches us back to an approach where we use the Pod "exec" API to grab the keys we need, rather than forcing our code to run on the control plane node. It will help us fail gracefully (or dynamically switch to alternate implementations) when the cluster is not self-hosted. Signed-off-by: Matt Moyer <moyerm@vmware.com> Co-authored-by: Ryan Richard <richardry@vmware.com>
This commit is contained in:
parent
40d1360b74
commit
1b9a70d089
@ -46,13 +46,14 @@ data:
|
|||||||
#! TODO set up healthy, ready, etc. probes correctly?
|
#! TODO set up healthy, ready, etc. probes correctly?
|
||||||
#! TODO set resource minimums (e.g. 512MB RAM) to make sure we get scheduled onto a reasonable node?
|
#! TODO set resource minimums (e.g. 512MB RAM) to make sure we get scheduled onto a reasonable node?
|
||||||
apiVersion: apps/v1
|
apiVersion: apps/v1
|
||||||
kind: DaemonSet
|
kind: Deployment
|
||||||
metadata:
|
metadata:
|
||||||
name: #@ data.values.app_name
|
name: #@ data.values.app_name
|
||||||
namespace: #@ data.values.namespace
|
namespace: #@ data.values.namespace
|
||||||
labels:
|
labels:
|
||||||
app: #@ data.values.app_name
|
app: #@ data.values.app_name
|
||||||
spec:
|
spec:
|
||||||
|
replicas: 2
|
||||||
selector:
|
selector:
|
||||||
matchLabels:
|
matchLabels:
|
||||||
app: #@ data.values.app_name
|
app: #@ data.values.app_name
|
||||||
@ -79,15 +80,11 @@ spec:
|
|||||||
args:
|
args:
|
||||||
- --config=/etc/config/placeholder-name.yaml
|
- --config=/etc/config/placeholder-name.yaml
|
||||||
- --downward-api-path=/etc/podinfo
|
- --downward-api-path=/etc/podinfo
|
||||||
- --cluster-signing-cert-file=/etc/kubernetes/pki/ca.crt
|
|
||||||
- --cluster-signing-key-file=/etc/kubernetes/pki/ca.key
|
|
||||||
volumeMounts:
|
volumeMounts:
|
||||||
- name: config-volume
|
- name: config-volume
|
||||||
mountPath: /etc/config
|
mountPath: /etc/config
|
||||||
- name: podinfo
|
- name: podinfo
|
||||||
mountPath: /etc/podinfo
|
mountPath: /etc/podinfo
|
||||||
- name: k8s-certs
|
|
||||||
mountPath: /etc/kubernetes/pki
|
|
||||||
livenessProbe:
|
livenessProbe:
|
||||||
httpGet:
|
httpGet:
|
||||||
path: /healthz
|
path: /healthz
|
||||||
@ -119,16 +116,10 @@ spec:
|
|||||||
- path: "namespace"
|
- path: "namespace"
|
||||||
fieldRef:
|
fieldRef:
|
||||||
fieldPath: metadata.namespace
|
fieldPath: metadata.namespace
|
||||||
- name: k8s-certs
|
|
||||||
hostPath:
|
|
||||||
path: /etc/kubernetes/pki
|
|
||||||
type: DirectoryOrCreate
|
|
||||||
nodeSelector: #! Create Pods on all nodes which match this node selector, and not on any other nodes.
|
|
||||||
node-role.kubernetes.io/master: ""
|
|
||||||
tolerations:
|
tolerations:
|
||||||
- key: CriticalAddonsOnly
|
- key: CriticalAddonsOnly
|
||||||
operator: Exists
|
operator: Exists
|
||||||
- key: node-role.kubernetes.io/master #! Allow running on master nodes.
|
- key: node-role.kubernetes.io/master #! Allow running on master nodes too
|
||||||
effect: NoSchedule
|
effect: NoSchedule
|
||||||
#! "system-cluster-critical" cannot be used outside the kube-system namespace until Kubernetes >= 1.17,
|
#! "system-cluster-critical" cannot be used outside the kube-system namespace until Kubernetes >= 1.17,
|
||||||
#! so we skip setting this for now (see https://github.com/kubernetes/kubernetes/issues/60596).
|
#! so we skip setting this for now (see https://github.com/kubernetes/kubernetes/issues/60596).
|
||||||
|
@ -62,6 +62,35 @@ roleRef:
|
|||||||
name: #@ data.values.app_name + "-aggregated-api-server-role"
|
name: #@ data.values.app_name + "-aggregated-api-server-role"
|
||||||
apiGroup: rbac.authorization.k8s.io
|
apiGroup: rbac.authorization.k8s.io
|
||||||
|
|
||||||
|
#! Give permission to list pods and pod exec in the kube-system namespace so we can find the API server's private key
|
||||||
|
---
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: Role
|
||||||
|
metadata:
|
||||||
|
name: #@ data.values.app_name + "-kube-system-pod-exec-role"
|
||||||
|
namespace: kube-system
|
||||||
|
rules:
|
||||||
|
- apiGroups: [""]
|
||||||
|
resources: [pods]
|
||||||
|
verbs: [get, list]
|
||||||
|
- apiGroups: [""]
|
||||||
|
resources: [pods/exec]
|
||||||
|
verbs: [create]
|
||||||
|
---
|
||||||
|
kind: RoleBinding
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
metadata:
|
||||||
|
name: #@ data.values.app_name + "-kube-system-pod-exec-role-binding"
|
||||||
|
namespace: kube-system
|
||||||
|
subjects:
|
||||||
|
- kind: ServiceAccount
|
||||||
|
name: #@ data.values.app_name + "-service-account"
|
||||||
|
namespace: #@ data.values.namespace
|
||||||
|
roleRef:
|
||||||
|
kind: Role
|
||||||
|
name: #@ data.values.app_name + "-kube-system-pod-exec-role"
|
||||||
|
apiGroup: rbac.authorization.k8s.io
|
||||||
|
|
||||||
#! Allow both authenticated and unauthenticated CredentialRequests (i.e. allow all requests)
|
#! Allow both authenticated and unauthenticated CredentialRequests (i.e. allow all requests)
|
||||||
---
|
---
|
||||||
apiVersion: rbac.authorization.k8s.io/v1
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
1
go.mod
1
go.mod
@ -9,6 +9,7 @@ require (
|
|||||||
github.com/google/go-cmp v0.5.0
|
github.com/google/go-cmp v0.5.0
|
||||||
github.com/sclevine/spec v1.4.0
|
github.com/sclevine/spec v1.4.0
|
||||||
github.com/spf13/cobra v1.0.0
|
github.com/spf13/cobra v1.0.0
|
||||||
|
github.com/spf13/pflag v1.0.5
|
||||||
github.com/stretchr/testify v1.6.1
|
github.com/stretchr/testify v1.6.1
|
||||||
github.com/suzerain-io/controller-go v0.0.0-20200730212956-7f99b569ca9f
|
github.com/suzerain-io/controller-go v0.0.0-20200730212956-7f99b569ca9f
|
||||||
github.com/suzerain-io/placeholder-name/kubernetes/1.19/api v0.0.0-00010101000000-000000000000
|
github.com/suzerain-io/placeholder-name/kubernetes/1.19/api v0.0.0-00010101000000-000000000000
|
||||||
|
1
go.sum
1
go.sum
@ -100,6 +100,7 @@ github.com/denis-tingajkin/go-header v0.3.1/go.mod h1:sq/2IxMhaZX+RRcgHfCRx/m0M5
|
|||||||
github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
|
github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
|
||||||
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
|
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
|
||||||
github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
|
github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
|
||||||
|
github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96 h1:cenwrSVm+Z7QLSV/BsnenAOcDXdX4cMv4wP0B/5QbPg=
|
||||||
github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
|
github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
|
||||||
github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
|
github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
|
||||||
github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
|
github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
|
||||||
|
@ -62,8 +62,8 @@ func secureEnv() env {
|
|||||||
var ErrInvalidCACertificate = fmt.Errorf("invalid CA certificate")
|
var ErrInvalidCACertificate = fmt.Errorf("invalid CA certificate")
|
||||||
|
|
||||||
// Load a certificate authority from an existing certificate and private key (in PEM format).
|
// Load a certificate authority from an existing certificate and private key (in PEM format).
|
||||||
func Load(certPath string, keyPath string) (*CA, error) {
|
func Load(certPEM string, keyPEM string) (*CA, error) {
|
||||||
cert, err := tls.LoadX509KeyPair(certPath, keyPath)
|
cert, err := tls.X509KeyPair([]byte(certPEM), []byte(keyPEM))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("could not load CA: %w", err)
|
return nil, fmt.Errorf("could not load CA: %w", err)
|
||||||
}
|
}
|
||||||
|
@ -12,6 +12,7 @@ import (
|
|||||||
"crypto/x509/pkix"
|
"crypto/x509/pkix"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
|
"io/ioutil"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
@ -19,6 +20,19 @@ import (
|
|||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
func loadFromFiles(t *testing.T, certPath string, keyPath string) (*CA, error) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
certPEM, err := ioutil.ReadFile(certPath)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
keyPEM, err := ioutil.ReadFile(keyPath)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
ca, err := Load(string(certPEM), string(keyPEM))
|
||||||
|
return ca, err
|
||||||
|
}
|
||||||
|
|
||||||
func TestLoad(t *testing.T) {
|
func TestLoad(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
@ -26,30 +40,6 @@ func TestLoad(t *testing.T) {
|
|||||||
keyPath string
|
keyPath string
|
||||||
wantErr string
|
wantErr string
|
||||||
}{
|
}{
|
||||||
{
|
|
||||||
name: "missing cert",
|
|
||||||
certPath: "./testdata/cert-does-not-exist",
|
|
||||||
keyPath: "./testdata/test.key",
|
|
||||||
wantErr: "could not load CA: open ./testdata/cert-does-not-exist: no such file or directory",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "empty cert",
|
|
||||||
certPath: "./testdata/empty",
|
|
||||||
keyPath: "./testdata/test.key",
|
|
||||||
wantErr: "could not load CA: tls: failed to find any PEM data in certificate input",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "invalid cert",
|
|
||||||
certPath: "./testdata/invalid",
|
|
||||||
keyPath: "./testdata/test.key",
|
|
||||||
wantErr: "could not load CA: tls: failed to find any PEM data in certificate input",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "missing key",
|
|
||||||
certPath: "./testdata/test.crt",
|
|
||||||
keyPath: "./testdata/key-does-not-exist",
|
|
||||||
wantErr: "could not load CA: open ./testdata/key-does-not-exist: no such file or directory",
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
name: "empty key",
|
name: "empty key",
|
||||||
certPath: "./testdata/test.crt",
|
certPath: "./testdata/test.crt",
|
||||||
@ -83,7 +73,7 @@ func TestLoad(t *testing.T) {
|
|||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
tt := tt
|
tt := tt
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
ca, err := Load(tt.certPath, tt.keyPath)
|
ca, err := loadFromFiles(t, tt.certPath, tt.keyPath)
|
||||||
if tt.wantErr != "" {
|
if tt.wantErr != "" {
|
||||||
require.EqualError(t, err, tt.wantErr)
|
require.EqualError(t, err, tt.wantErr)
|
||||||
return
|
return
|
||||||
@ -202,7 +192,7 @@ func (e *errSigner) Sign(_ io.Reader, _ []byte, _ crypto.SignerOpts) ([]byte, er
|
|||||||
func TestIssue(t *testing.T) {
|
func TestIssue(t *testing.T) {
|
||||||
now := time.Date(2020, 7, 10, 12, 41, 12, 1234, time.UTC)
|
now := time.Date(2020, 7, 10, 12, 41, 12, 1234, time.UTC)
|
||||||
|
|
||||||
realCA, err := Load("./testdata/test.crt", "./testdata/test.key")
|
realCA, err := loadFromFiles(t, "./testdata/test.crt", "./testdata/test.key")
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
@ -302,7 +292,7 @@ func TestIssue(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestIssuePEM(t *testing.T) {
|
func TestIssuePEM(t *testing.T) {
|
||||||
realCA, err := Load("./testdata/test.crt", "./testdata/test.key")
|
realCA, err := loadFromFiles(t, "./testdata/test.crt", "./testdata/test.key")
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
certPEM, keyPEM, err := realCA.IssuePEM(pkix.Name{CommonName: "Test Server"}, []string{"example.com"}, 10*time.Minute)
|
certPEM, keyPEM, err := realCA.IssuePEM(pkix.Name{CommonName: "Test Server"}, []string{"example.com"}, 10*time.Minute)
|
||||||
|
209
internal/certauthority/kubecertauthority/kubecertauthority.go
Normal file
209
internal/certauthority/kubecertauthority/kubecertauthority.go
Normal file
@ -0,0 +1,209 @@
|
|||||||
|
/*
|
||||||
|
Copyright 2020 VMware, Inc.
|
||||||
|
SPDX-License-Identifier: Apache-2.0
|
||||||
|
*/
|
||||||
|
|
||||||
|
// Package kubecertauthority implements a signer backed by the kubernetes controller-manager signing
|
||||||
|
// keys (accessed via the kubernetes Exec API).
|
||||||
|
package kubecertauthority
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"crypto/x509/pkix"
|
||||||
|
"fmt"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/spf13/pflag"
|
||||||
|
v1 "k8s.io/api/core/v1"
|
||||||
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||||
|
"k8s.io/client-go/deprecated/scheme"
|
||||||
|
"k8s.io/client-go/kubernetes"
|
||||||
|
restclient "k8s.io/client-go/rest"
|
||||||
|
"k8s.io/client-go/tools/remotecommand"
|
||||||
|
"k8s.io/klog/v2"
|
||||||
|
|
||||||
|
"github.com/suzerain-io/placeholder-name/internal/certauthority"
|
||||||
|
"github.com/suzerain-io/placeholder-name/internal/constable"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ErrNoKubeControllerManagerPod is returned when no kube-controller-manager pod is found on the cluster.
|
||||||
|
const ErrNoKubeControllerManagerPod = constable.Error("did not find kube-controller-manager pod")
|
||||||
|
|
||||||
|
const k8sAPIServerCACertPEMDefaultPath = "/etc/kubernetes/ca/ca.pem"
|
||||||
|
const k8sAPIServerCAKeyPEMDefaultPath = "/etc/kubernetes/ca/ca.key"
|
||||||
|
|
||||||
|
type signer interface {
|
||||||
|
IssuePEM(subject pkix.Name, dnsNames []string, ttl time.Duration) ([]byte, []byte, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
type PodCommandExecutor interface {
|
||||||
|
Exec(podNamespace string, podName string, commandAndArgs ...string) (stdoutResult string, err error)
|
||||||
|
}
|
||||||
|
|
||||||
|
type kubeClientPodCommandExecutor struct {
|
||||||
|
kubeConfig *restclient.Config
|
||||||
|
kubeClient kubernetes.Interface
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewPodCommandExecutor(kubeConfig *restclient.Config, kubeClient kubernetes.Interface) PodCommandExecutor {
|
||||||
|
return &kubeClientPodCommandExecutor{kubeConfig: kubeConfig, kubeClient: kubeClient}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *kubeClientPodCommandExecutor) Exec(podNamespace string, podName string, commandAndArgs ...string) (string, error) {
|
||||||
|
request := s.kubeClient.
|
||||||
|
CoreV1().
|
||||||
|
RESTClient().
|
||||||
|
Post().
|
||||||
|
Namespace(podNamespace).
|
||||||
|
Resource("pods").
|
||||||
|
Name(podName).
|
||||||
|
SubResource("exec").
|
||||||
|
VersionedParams(&v1.PodExecOptions{
|
||||||
|
Stdin: false,
|
||||||
|
Stdout: true,
|
||||||
|
Stderr: false,
|
||||||
|
TTY: false,
|
||||||
|
Command: commandAndArgs,
|
||||||
|
}, scheme.ParameterCodec)
|
||||||
|
|
||||||
|
executor, err := remotecommand.NewSPDYExecutor(s.kubeConfig, "POST", request.URL())
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
var stdoutBuf bytes.Buffer
|
||||||
|
if err := executor.Stream(remotecommand.StreamOptions{Stdout: &stdoutBuf}); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return stdoutBuf.String(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type CA struct {
|
||||||
|
kubeClient kubernetes.Interface
|
||||||
|
podCommandExecutor PodCommandExecutor
|
||||||
|
|
||||||
|
shutdown, done chan struct{}
|
||||||
|
|
||||||
|
lock sync.RWMutex
|
||||||
|
activeSigner signer
|
||||||
|
}
|
||||||
|
|
||||||
|
type ShutdownFunc func()
|
||||||
|
|
||||||
|
// New creates a new instance of a CA which is has loaded the kube API server's private key
|
||||||
|
// and is ready to issue certs, or an error. When successful, it also starts a goroutine
|
||||||
|
// to periodically reload the kube API server's private key in case it changed, and returns
|
||||||
|
// a function that can be used to shut down that goroutine.
|
||||||
|
func New(kubeClient kubernetes.Interface, podCommandExecutor PodCommandExecutor, tick <-chan time.Time) (*CA, ShutdownFunc, error) {
|
||||||
|
signer, err := createSignerWithAPIServerSecret(kubeClient, podCommandExecutor)
|
||||||
|
if err != nil {
|
||||||
|
// The initial load failed, so give up
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
result := &CA{
|
||||||
|
kubeClient: kubeClient,
|
||||||
|
podCommandExecutor: podCommandExecutor,
|
||||||
|
activeSigner: signer,
|
||||||
|
shutdown: make(chan struct{}),
|
||||||
|
done: make(chan struct{}),
|
||||||
|
}
|
||||||
|
go result.refreshLoop(tick)
|
||||||
|
return result, result.shutdownRefresh, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func createSignerWithAPIServerSecret(kubeClient kubernetes.Interface, podCommandExecutor PodCommandExecutor) (signer, error) {
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
pod, err := findControllerManagerPod(ctx, kubeClient)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
certPath, keyPath := getKeypairFilePaths(pod)
|
||||||
|
|
||||||
|
certPEM, err := podCommandExecutor.Exec(pod.Namespace, pod.Name, "cat", certPath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
keyPEM, err := podCommandExecutor.Exec(pod.Namespace, pod.Name, "cat", keyPath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return certauthority.Load(certPEM, keyPEM)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *CA) refreshLoop(tick <-chan time.Time) {
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-c.shutdown:
|
||||||
|
close(c.done)
|
||||||
|
return
|
||||||
|
case <-tick:
|
||||||
|
c.updateSigner()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *CA) updateSigner() {
|
||||||
|
newSigner, err := createSignerWithAPIServerSecret(c.kubeClient, c.podCommandExecutor)
|
||||||
|
if err != nil {
|
||||||
|
klog.Errorf("could not create signer with API server secret: %s", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
c.lock.Lock()
|
||||||
|
c.activeSigner = newSigner
|
||||||
|
c.lock.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *CA) shutdownRefresh() {
|
||||||
|
close(c.shutdown)
|
||||||
|
<-c.done
|
||||||
|
}
|
||||||
|
|
||||||
|
// IssuePEM issues a new server certificate for the given identity and duration, returning it as a pair of
|
||||||
|
// PEM-formatted byte slices for the certificate and private key.
|
||||||
|
func (c *CA) IssuePEM(subject pkix.Name, dnsNames []string, ttl time.Duration) ([]byte, []byte, error) {
|
||||||
|
c.lock.RLock()
|
||||||
|
signer := c.activeSigner
|
||||||
|
c.lock.RUnlock()
|
||||||
|
|
||||||
|
return signer.IssuePEM(subject, dnsNames, ttl)
|
||||||
|
}
|
||||||
|
|
||||||
|
func findControllerManagerPod(ctx context.Context, kubeClient kubernetes.Interface) (*v1.Pod, error) {
|
||||||
|
pods, err := kubeClient.CoreV1().Pods("kube-system").List(ctx, metav1.ListOptions{
|
||||||
|
LabelSelector: "component=kube-controller-manager",
|
||||||
|
FieldSelector: "status.phase=Running",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("could not check for kube-controller-manager pod: %w", err)
|
||||||
|
}
|
||||||
|
for _, pod := range pods.Items {
|
||||||
|
return &pod, nil
|
||||||
|
}
|
||||||
|
return nil, ErrNoKubeControllerManagerPod
|
||||||
|
}
|
||||||
|
|
||||||
|
func getKeypairFilePaths(pod *v1.Pod) (string, string) {
|
||||||
|
certPath := getContainerArgByName(pod, "cluster-signing-cert-file", k8sAPIServerCACertPEMDefaultPath)
|
||||||
|
keyPath := getContainerArgByName(pod, "cluster-signing-key-file", k8sAPIServerCAKeyPEMDefaultPath)
|
||||||
|
return certPath, keyPath
|
||||||
|
}
|
||||||
|
|
||||||
|
func getContainerArgByName(pod *v1.Pod, name string, defaultValue string) string {
|
||||||
|
for _, container := range pod.Spec.Containers {
|
||||||
|
flagset := pflag.NewFlagSet("", pflag.ContinueOnError)
|
||||||
|
flagset.ParseErrorsWhitelist = pflag.ParseErrorsWhitelist{UnknownFlags: true}
|
||||||
|
var val string
|
||||||
|
flagset.StringVar(&val, name, "", "")
|
||||||
|
_ = flagset.Parse(append(container.Command, container.Args...))
|
||||||
|
if val != "" {
|
||||||
|
return val
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return defaultValue
|
||||||
|
}
|
@ -0,0 +1,329 @@
|
|||||||
|
/*
|
||||||
|
Copyright 2020 VMware, Inc.
|
||||||
|
SPDX-License-Identifier: Apache-2.0
|
||||||
|
*/
|
||||||
|
|
||||||
|
package kubecertauthority
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/x509"
|
||||||
|
"crypto/x509/pkix"
|
||||||
|
"encoding/pem"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io/ioutil"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/sclevine/spec"
|
||||||
|
"github.com/sclevine/spec/report"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
corev1 "k8s.io/api/core/v1"
|
||||||
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||||
|
kubernetesfake "k8s.io/client-go/kubernetes/fake"
|
||||||
|
"k8s.io/klog/v2"
|
||||||
|
|
||||||
|
"github.com/suzerain-io/placeholder-name/internal/testutil"
|
||||||
|
)
|
||||||
|
|
||||||
|
type fakePodExecutor struct {
|
||||||
|
resultsToReturn []string
|
||||||
|
errorsToReturn []error
|
||||||
|
|
||||||
|
calledWithPodName []string
|
||||||
|
calledWithPodNamespace []string
|
||||||
|
calledWithCommandAndArgs [][]string
|
||||||
|
|
||||||
|
callCount int
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *fakePodExecutor) Exec(podNamespace string, podName string, commandAndArgs ...string) (string, error) {
|
||||||
|
s.calledWithPodNamespace = append(s.calledWithPodNamespace, podNamespace)
|
||||||
|
s.calledWithPodName = append(s.calledWithPodName, podName)
|
||||||
|
s.calledWithCommandAndArgs = append(s.calledWithCommandAndArgs, commandAndArgs)
|
||||||
|
result := s.resultsToReturn[s.callCount]
|
||||||
|
var err error = nil
|
||||||
|
if s.errorsToReturn != nil {
|
||||||
|
err = s.errorsToReturn[s.callCount]
|
||||||
|
}
|
||||||
|
s.callCount++
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCA(t *testing.T) {
|
||||||
|
spec.Run(t, "CA", func(t *testing.T, when spec.G, it spec.S) {
|
||||||
|
var r *require.Assertions
|
||||||
|
var fakeCertPEM, fakeKeyPEM string
|
||||||
|
var fakeCert2PEM, fakeKey2PEM string
|
||||||
|
var fakePod *corev1.Pod
|
||||||
|
var kubeAPIClient *kubernetesfake.Clientset
|
||||||
|
var fakeExecutor *fakePodExecutor
|
||||||
|
var neverTicker <-chan time.Time
|
||||||
|
|
||||||
|
var logger *testutil.TranscriptLogger
|
||||||
|
|
||||||
|
it.Before(func() {
|
||||||
|
r = require.New(t)
|
||||||
|
|
||||||
|
loadFile := func(filename string) string {
|
||||||
|
bytes, err := ioutil.ReadFile(filename)
|
||||||
|
r.NoError(err)
|
||||||
|
return string(bytes)
|
||||||
|
}
|
||||||
|
fakeCertPEM = loadFile("./testdata/test.crt")
|
||||||
|
fakeKeyPEM = loadFile("./testdata/test.key")
|
||||||
|
fakeCert2PEM = loadFile("./testdata/test2.crt")
|
||||||
|
fakeKey2PEM = loadFile("./testdata/test2.key")
|
||||||
|
|
||||||
|
fakePod = &corev1.Pod{
|
||||||
|
TypeMeta: metav1.TypeMeta{},
|
||||||
|
ObjectMeta: metav1.ObjectMeta{
|
||||||
|
Name: "fake-pod",
|
||||||
|
Namespace: "kube-system",
|
||||||
|
Labels: map[string]string{"component": "kube-controller-manager"},
|
||||||
|
},
|
||||||
|
Spec: corev1.PodSpec{
|
||||||
|
Containers: []corev1.Container{{Name: "kube-controller-manager"}},
|
||||||
|
},
|
||||||
|
Status: corev1.PodStatus{
|
||||||
|
Phase: "Running",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
kubeAPIClient = kubernetesfake.NewSimpleClientset()
|
||||||
|
|
||||||
|
fakeExecutor = &fakePodExecutor{
|
||||||
|
resultsToReturn: []string{
|
||||||
|
fakeCertPEM,
|
||||||
|
fakeKeyPEM,
|
||||||
|
fakeCert2PEM,
|
||||||
|
fakeKey2PEM,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
logger = testutil.NewTranscriptLogger(t)
|
||||||
|
klog.SetLogger(logger) // this is unfortunately a global logger, so can't run these tests in parallel :(
|
||||||
|
})
|
||||||
|
|
||||||
|
it.After(func() {
|
||||||
|
klog.SetLogger(nil)
|
||||||
|
})
|
||||||
|
|
||||||
|
when("the kube-controller-manager pod is found with default CLI flag values", func() {
|
||||||
|
it.Before(func() {
|
||||||
|
err := kubeAPIClient.Tracker().Add(fakePod)
|
||||||
|
r.NoError(err)
|
||||||
|
})
|
||||||
|
|
||||||
|
when("the exec commands return the API server's keypair", func() {
|
||||||
|
it("finds the API server's signing key and uses it to issue certificates", func() {
|
||||||
|
fakeTicker := make(chan time.Time)
|
||||||
|
|
||||||
|
subject, shutdownFunc, err := New(kubeAPIClient, fakeExecutor, fakeTicker)
|
||||||
|
r.NoError(err)
|
||||||
|
r.NotNil(shutdownFunc)
|
||||||
|
defer shutdownFunc()
|
||||||
|
|
||||||
|
r.Equal(2, fakeExecutor.callCount)
|
||||||
|
|
||||||
|
r.Equal("kube-system", fakeExecutor.calledWithPodNamespace[0])
|
||||||
|
r.Equal("fake-pod", fakeExecutor.calledWithPodName[0])
|
||||||
|
r.Equal([]string{"cat", "/etc/kubernetes/ca/ca.pem"}, fakeExecutor.calledWithCommandAndArgs[0])
|
||||||
|
|
||||||
|
r.Equal("kube-system", fakeExecutor.calledWithPodNamespace[1])
|
||||||
|
r.Equal("fake-pod", fakeExecutor.calledWithPodName[1])
|
||||||
|
r.Equal([]string{"cat", "/etc/kubernetes/ca/ca.key"}, fakeExecutor.calledWithCommandAndArgs[1])
|
||||||
|
|
||||||
|
// Validate that we can issue a certificate signed by the original API server CA.
|
||||||
|
certPEM, keyPEM, err := subject.IssuePEM(
|
||||||
|
pkix.Name{CommonName: "Test Server"},
|
||||||
|
[]string{"example.com"},
|
||||||
|
10*time.Minute,
|
||||||
|
)
|
||||||
|
r.NoError(err)
|
||||||
|
validCert := testutil.ValidateCertificate(t, fakeCertPEM, string(certPEM))
|
||||||
|
validCert.RequireDNSName("example.com")
|
||||||
|
validCert.RequireLifetime(time.Now(), time.Now().Add(10*time.Minute), 2*time.Minute)
|
||||||
|
validCert.RequireMatchesPrivateKey(string(keyPEM))
|
||||||
|
|
||||||
|
// Tick the timer and wait for another refresh loop to complete.
|
||||||
|
fakeTicker <- time.Now()
|
||||||
|
|
||||||
|
var secondCertPEM, secondKeyPEM string
|
||||||
|
r.Eventually(func() bool {
|
||||||
|
certPEM, keyPEM, err := subject.IssuePEM(
|
||||||
|
pkix.Name{CommonName: "Test Server"},
|
||||||
|
[]string{"example.com"},
|
||||||
|
10*time.Minute,
|
||||||
|
)
|
||||||
|
r.NoError(err)
|
||||||
|
secondCertPEM = string(certPEM)
|
||||||
|
secondKeyPEM = string(keyPEM)
|
||||||
|
|
||||||
|
block, _ := pem.Decode(certPEM)
|
||||||
|
require.NotNil(t, block)
|
||||||
|
parsed, err := x509.ParseCertificate(block.Bytes)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
// Validate the created cert using the second API server CA.
|
||||||
|
roots := x509.NewCertPool()
|
||||||
|
require.True(t, roots.AppendCertsFromPEM([]byte(fakeCert2PEM)))
|
||||||
|
opts := x509.VerifyOptions{Roots: roots}
|
||||||
|
_, err = parsed.Verify(opts)
|
||||||
|
return err == nil
|
||||||
|
}, 5*time.Second, 100*time.Millisecond)
|
||||||
|
|
||||||
|
validCert2 := testutil.ValidateCertificate(t, fakeCert2PEM, secondCertPEM)
|
||||||
|
validCert2.RequireDNSName("example.com")
|
||||||
|
validCert2.RequireLifetime(time.Now(), time.Now().Add(10*time.Minute), 2*time.Minute)
|
||||||
|
validCert2.RequireMatchesPrivateKey(secondKeyPEM)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
when("the exec commands return the API server's keypair the first time but subsequently fails", func() {
|
||||||
|
it.Before(func() {
|
||||||
|
fakeExecutor.errorsToReturn = []error{nil, nil, fmt.Errorf("some exec error")}
|
||||||
|
})
|
||||||
|
|
||||||
|
it("logs an error message", func() {
|
||||||
|
fakeTicker := make(chan time.Time)
|
||||||
|
|
||||||
|
subject, shutdownFunc, err := New(kubeAPIClient, fakeExecutor, fakeTicker)
|
||||||
|
r.NoError(err)
|
||||||
|
r.NotNil(shutdownFunc)
|
||||||
|
defer shutdownFunc()
|
||||||
|
r.Equal(2, fakeExecutor.callCount)
|
||||||
|
|
||||||
|
// Tick the timer and wait for another refresh loop to complete.
|
||||||
|
fakeTicker <- time.Now()
|
||||||
|
|
||||||
|
// Wait for there to be a log output and require that it matches our expectation.
|
||||||
|
r.Eventually(func() bool { return len(logger.Transcript()) >= 1 }, 5*time.Second, 10*time.Millisecond)
|
||||||
|
r.Contains(logger.Transcript()[0].Message, "could not create signer with API server secret: some exec error")
|
||||||
|
r.Equal(logger.Transcript()[0].Level, "error")
|
||||||
|
|
||||||
|
// Validate that we can still issue a certificate signed by the original API server CA.
|
||||||
|
certPEM, _, err := subject.IssuePEM(
|
||||||
|
pkix.Name{CommonName: "Test Server"},
|
||||||
|
[]string{"example.com"},
|
||||||
|
10*time.Minute,
|
||||||
|
)
|
||||||
|
r.NoError(err)
|
||||||
|
testutil.ValidateCertificate(t, fakeCertPEM, string(certPEM))
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
when("the exec commands succeed but return garbage", func() {
|
||||||
|
it.Before(func() {
|
||||||
|
fakeExecutor.resultsToReturn = []string{"not a cert", "not a private key"}
|
||||||
|
})
|
||||||
|
|
||||||
|
it("returns an error", func() {
|
||||||
|
subject, shutdownFunc, err := New(kubeAPIClient, fakeExecutor, neverTicker)
|
||||||
|
r.Nil(subject)
|
||||||
|
r.Nil(shutdownFunc)
|
||||||
|
r.EqualError(err, "could not load CA: tls: failed to find any PEM data in certificate input")
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
when("the first exec command returns an error", func() {
|
||||||
|
it.Before(func() {
|
||||||
|
fakeExecutor.errorsToReturn = []error{fmt.Errorf("some error"), nil}
|
||||||
|
})
|
||||||
|
|
||||||
|
it("returns an error", func() {
|
||||||
|
subject, shutdownFunc, err := New(kubeAPIClient, fakeExecutor, neverTicker)
|
||||||
|
r.Nil(subject)
|
||||||
|
r.Nil(shutdownFunc)
|
||||||
|
r.EqualError(err, "some error")
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
when("the second exec command returns an error", func() {
|
||||||
|
it.Before(func() {
|
||||||
|
fakeExecutor.errorsToReturn = []error{nil, fmt.Errorf("some error")}
|
||||||
|
})
|
||||||
|
|
||||||
|
it("returns an error", func() {
|
||||||
|
subject, shutdownFunc, err := New(kubeAPIClient, fakeExecutor, neverTicker)
|
||||||
|
r.Nil(subject)
|
||||||
|
r.Nil(shutdownFunc)
|
||||||
|
r.EqualError(err, "some error")
|
||||||
|
})
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
when("the kube-controller-manager pod is found with non-default CLI flag values", func() {
|
||||||
|
it.Before(func() {
|
||||||
|
fakePod.Spec.Containers[0].Command = []string{
|
||||||
|
"kube-controller-manager",
|
||||||
|
"--cluster-signing-cert-file=/etc/kubernetes/ca/non-default.pem",
|
||||||
|
}
|
||||||
|
fakePod.Spec.Containers[0].Args = []string{
|
||||||
|
"--cluster-signing-key-file=/etc/kubernetes/ca/non-default.key",
|
||||||
|
}
|
||||||
|
err := kubeAPIClient.Tracker().Add(fakePod)
|
||||||
|
r.NoError(err)
|
||||||
|
})
|
||||||
|
|
||||||
|
it("finds the API server's signing key and uses it to issue certificates", func() {
|
||||||
|
_, shutdownFunc, err := New(kubeAPIClient, fakeExecutor, neverTicker)
|
||||||
|
r.NoError(err)
|
||||||
|
r.NotNil(shutdownFunc)
|
||||||
|
defer shutdownFunc()
|
||||||
|
|
||||||
|
r.Equal(2, fakeExecutor.callCount)
|
||||||
|
|
||||||
|
r.Equal("kube-system", fakeExecutor.calledWithPodNamespace[0])
|
||||||
|
r.Equal("fake-pod", fakeExecutor.calledWithPodName[0])
|
||||||
|
r.Equal([]string{"cat", "/etc/kubernetes/ca/non-default.pem"}, fakeExecutor.calledWithCommandAndArgs[0])
|
||||||
|
|
||||||
|
r.Equal("kube-system", fakeExecutor.calledWithPodNamespace[1])
|
||||||
|
r.Equal("fake-pod", fakeExecutor.calledWithPodName[1])
|
||||||
|
r.Equal([]string{"cat", "/etc/kubernetes/ca/non-default.key"}, fakeExecutor.calledWithCommandAndArgs[1])
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
when("the kube-controller-manager pod is found with non-default CLI flag values separated by spaces", func() {
|
||||||
|
it.Before(func() {
|
||||||
|
fakePod.Spec.Containers[0].Command = []string{
|
||||||
|
"kube-controller-manager",
|
||||||
|
"--cluster-signing-cert-file", "/etc/kubernetes/ca/non-default.pem",
|
||||||
|
"--cluster-signing-key-file", "/etc/kubernetes/ca/non-default.key",
|
||||||
|
"--foo=bar",
|
||||||
|
}
|
||||||
|
err := kubeAPIClient.Tracker().Add(fakePod)
|
||||||
|
r.NoError(err)
|
||||||
|
})
|
||||||
|
|
||||||
|
it("finds the API server's signing key and uses it to issue certificates", func() {
|
||||||
|
_, shutdownFunc, err := New(kubeAPIClient, fakeExecutor, neverTicker)
|
||||||
|
r.NoError(err)
|
||||||
|
r.NotNil(shutdownFunc)
|
||||||
|
defer shutdownFunc()
|
||||||
|
|
||||||
|
r.Equal(2, fakeExecutor.callCount)
|
||||||
|
|
||||||
|
r.Equal("kube-system", fakeExecutor.calledWithPodNamespace[0])
|
||||||
|
r.Equal("fake-pod", fakeExecutor.calledWithPodName[0])
|
||||||
|
r.Equal([]string{"cat", "/etc/kubernetes/ca/non-default.pem"}, fakeExecutor.calledWithCommandAndArgs[0])
|
||||||
|
|
||||||
|
r.Equal("kube-system", fakeExecutor.calledWithPodNamespace[1])
|
||||||
|
r.Equal("fake-pod", fakeExecutor.calledWithPodName[1])
|
||||||
|
r.Equal([]string{"cat", "/etc/kubernetes/ca/non-default.key"}, fakeExecutor.calledWithCommandAndArgs[1])
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
when("the kube-controller-manager pod is not found", func() {
|
||||||
|
it("returns an error", func() {
|
||||||
|
subject, shutdownFunc, err := New(kubeAPIClient, fakeExecutor, neverTicker)
|
||||||
|
r.Nil(subject)
|
||||||
|
r.Nil(shutdownFunc)
|
||||||
|
r.True(errors.Is(err, ErrNoKubeControllerManagerPod))
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}, spec.Report(report.Terminal{}))
|
||||||
|
}
|
17
internal/certauthority/kubecertauthority/testdata/test.crt
vendored
Normal file
17
internal/certauthority/kubecertauthority/testdata/test.crt
vendored
Normal file
@ -0,0 +1,17 @@
|
|||||||
|
-----BEGIN CERTIFICATE-----
|
||||||
|
MIICyDCCAbCgAwIBAgIBADANBgkqhkiG9w0BAQsFADAVMRMwEQYDVQQDEwprdWJl
|
||||||
|
cm5ldGVzMB4XDTIwMDcyNTIxMDQxOFoXDTMwMDcyMzIxMDQxOFowFTETMBEGA1UE
|
||||||
|
AxMKa3ViZXJuZXRlczCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAL3K
|
||||||
|
hYv2gIQ1Dwzh2cWMid+ofAnvLIfV2Xv61vTLGprUI+XUqB4/gtf6X6UNn0Lett2n
|
||||||
|
d8p4wy7hw73hU/ggdvmWJvqBrSjc3JGfy+kj66fKXX+PTlbL7QbwiRvcSqIXIWlV
|
||||||
|
lHHxECWrED8jCulw/NVqfook/h5iNUCT9yswSJr/0fImiVnoTlIoEYG2eCNejZ5c
|
||||||
|
g39uD3ZTqd9ZxWwSLLnI+2kpJnZBPcd1ZQ8AQqzDgZtYRCqacn5gckQUKZWKQlxo
|
||||||
|
Eft6g1XHJouAWAZw7hEtk0v8rG0/eKF7wamxFi6BFVlbjWBsB4T9rApbdBWTKeCJ
|
||||||
|
Hv8fv5RMFSzpT3uzTO8CAwEAAaMjMCEwDgYDVR0PAQH/BAQDAgKkMA8GA1UdEwEB
|
||||||
|
/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBACh5RhbxqJe+Z/gc17cZhKNmdiwu
|
||||||
|
I2pLp3QBfwvN+Wbmajzw/7rYhY0d8JYVTJzXSCPWi6UAKxAtXOLF8WIIf9i39n6R
|
||||||
|
uKOBGW14FzzGyRJiD3qaG/JTvEW+SLhwl68Ndr5LHSnbugAqq31abcQy6Zl9v5A8
|
||||||
|
JKC97Lj/Sn8rj7opKy4W3oq7NCQsAb0zh4IllRF6UvSnJySfsg7xdXHHpxYDHtOS
|
||||||
|
XcOu5ySUIZTgFe9RfeUZlGZ5xn0ckMlQ7qW2Wx1q0OVWw5us4NtkGqKrHG4Tn1X7
|
||||||
|
uwo/Yytn5sDxrDv1/oii6AZOCsTPre4oD3wz4nmVzCVJcgrqH4Q24hT8WNg=
|
||||||
|
-----END CERTIFICATE-----
|
27
internal/certauthority/kubecertauthority/testdata/test.key
vendored
Normal file
27
internal/certauthority/kubecertauthority/testdata/test.key
vendored
Normal file
@ -0,0 +1,27 @@
|
|||||||
|
-----BEGIN RSA PRIVATE KEY-----
|
||||||
|
MIIEogIBAAKCAQEAvcqFi/aAhDUPDOHZxYyJ36h8Ce8sh9XZe/rW9MsamtQj5dSo
|
||||||
|
Hj+C1/pfpQ2fQt623ad3ynjDLuHDveFT+CB2+ZYm+oGtKNzckZ/L6SPrp8pdf49O
|
||||||
|
VsvtBvCJG9xKohchaVWUcfEQJasQPyMK6XD81Wp+iiT+HmI1QJP3KzBImv/R8iaJ
|
||||||
|
WehOUigRgbZ4I16NnlyDf24PdlOp31nFbBIsucj7aSkmdkE9x3VlDwBCrMOBm1hE
|
||||||
|
KppyfmByRBQplYpCXGgR+3qDVccmi4BYBnDuES2TS/ysbT94oXvBqbEWLoEVWVuN
|
||||||
|
YGwHhP2sClt0FZMp4Ike/x+/lEwVLOlPe7NM7wIDAQABAoIBAFC1tUEmHNUcM0BJ
|
||||||
|
M3D9KQzB+63F1mwVlx1QOOV1EeVR3co5Ox1R6PSr9sycFGQ9jgqI0zp5TJe9Tp6L
|
||||||
|
GkhklfPh1MWnK9o6wlnzWKXWrrp2Jni+mpPyuOPAmq4Maniv2XeP+0bROwqpyojv
|
||||||
|
AA7yC7M+TH226ZJGNVs3EV9+cwHml0yuzBfIJn/rv/w2g+WRKM/MC0S7k2d8bRlA
|
||||||
|
NycKVGAGBhKTltjoVYOeh6aHEpSjK8zfaePjo5dYJvoVIli60YCgcJOU/8jXT+Np
|
||||||
|
1Fm7tRvAtj3pUp0Sqdaf2RUzh9jfJp2VFCHuSJ6TPqArOyQojtMcTHF0TiW7xrHP
|
||||||
|
xOCRIAECgYEAwGBPU7vdthMJBg+ORUoGQQaItTeJvQwIqJvbKD2osp4jhS1dGZBw
|
||||||
|
W30GKEc/gd8JNtOq9BBnMicPF7hktuy+bSPv41XPud67rSSO7Tsw20C10gFRq06B
|
||||||
|
zIJWFAUqK3IkvVc3VDmtSLSDox4QZ/BdqaMlQ5y5JCsC5kThmkZFlO8CgYEA/I9X
|
||||||
|
YHi6RioMJE1fqOHJL4DDjlezmcuRrD7fE5InKbtJZ2JhGYOX/C0KXnHTOWTCDxxN
|
||||||
|
FBvpvD6Xv5o3PhB9Z6k2fqvJ4GS8urkG/KU4xcC+bak+9ava8oaiSqG16zD9NH2P
|
||||||
|
jJ60NrbLl1J0pU9fiwuFVUKJ4hDZOfN9RqYdyAECgYAVwo8WhJiGgM6zfcz073OX
|
||||||
|
pVqPTPHqjVLpZ3+5pIfRdGvGI6R1QM5EuvaYVb7MPOM47WZX5wcVOC/P2g6iVlMP
|
||||||
|
21HGIC2384a9BfaYxOo40q/+SiHnw6CQ9mkwKIllkqqvNA9RGpkMMUb2i28For2l
|
||||||
|
c4vCgxa6DZdtXns6TRqPxwKBgCfY5cxOv/T6BVhk7MbUeM2J31DB/ZAyUhV/Bess
|
||||||
|
kAlBh19MYk2IOZ6L7KriApV3lDaWHIMjtEkDByYvyq98Io0MYZCywfMpca10K+oI
|
||||||
|
l2B7/I+IuGpCZxUEsO5dfTpSTGDPvqpND9niFVUWqVi7oTNq6ep9yQtl5SADjqxq
|
||||||
|
4SABAoGAIm0hUg1wtcS46cGLy6PIkPM5tocTSghtz4vFsuk/i4QA9GBoBO2gH6ty
|
||||||
|
+kJHmeaXt2dmgySp0QAWit5UlceEumB0NXnAdJZQxeGSFSyYkDWhwXd8wDceKo/1
|
||||||
|
LfCU6Dk8IN/SsppVUWXQ2rlORvxlrHeCio8o0kS9Yiu55WMYg4g=
|
||||||
|
-----END RSA PRIVATE KEY-----
|
17
internal/certauthority/kubecertauthority/testdata/test2.crt
vendored
Normal file
17
internal/certauthority/kubecertauthority/testdata/test2.crt
vendored
Normal file
@ -0,0 +1,17 @@
|
|||||||
|
-----BEGIN CERTIFICATE-----
|
||||||
|
MIICyDCCAbCgAwIBAgIBADANBgkqhkiG9w0BAQsFADAVMRMwEQYDVQQDEwprdWJl
|
||||||
|
cm5ldGVzMB4XDTIwMDgxODE2NDEzNloXDTMwMDgxNjE2NDEzNlowFTETMBEGA1UE
|
||||||
|
AxMKa3ViZXJuZXRlczCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALH7
|
||||||
|
C2JpttDi3mxpD4bd+BZucCrS8XF2YwqYAr42HePp++PBnlUFqWmtPc9/bmo+7+7z
|
||||||
|
iAAlnAV0pJWP+HR/PskX8MRcFAA1HoXLa37Q4SuBBQG+JE+AeaOObmQYaCFv55ej
|
||||||
|
UF4+JIoQOdlbYEMYSI07el0cxQL4Io/CHJ3p7AtNElxjDuMK4B9W8NiCse3p7Uf+
|
||||||
|
Qje4we1TYOfcpAM0jpBPHK9vCBCpX+j52S5DUTRVIk9kye3lCDmWOXH/fhj/aJTM
|
||||||
|
1MP/hThbl2wIbFuv1bpa0kXNZs8xB63dtqROQ+lCghDmuayRmzwXl2PX6IgFFcjV
|
||||||
|
yAgjXrZqjihs+mY8eT0CAwEAAaMjMCEwDgYDVR0PAQH/BAQDAgKkMA8GA1UdEwEB
|
||||||
|
/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAE+Saqk2EyuIx1rxFWrOwpTi5q/B
|
||||||
|
p/TwEtrmrFIRVPnGeBnhyfbGXPDMkzIY1mEvztu8H+pm5RPyhQYLsuwzYiYMQyxX
|
||||||
|
yL9VvO7uydn7+3zX7oknQ5qAvN3nmItNyOKw3MRIKGsySNuTQ5JPtU/ufGlEivbK
|
||||||
|
vNaDBqjKrBvwhIKMdV9/xYSyeBhSSWr/6W1tAk+XbHhQH1M78rdwGN5SI75L4FGu
|
||||||
|
13kn/W2n8pE17TAY88B1YGKhsLSvf8KrFNYv+UUmzh2WstECKSlnbrSM+boMlGJn
|
||||||
|
XahE8M23fieB+SaenQdOezrY4GAnXQ3qToDlhdYAOkWhcGDct47VRM93whY=
|
||||||
|
-----END CERTIFICATE-----
|
27
internal/certauthority/kubecertauthority/testdata/test2.key
vendored
Normal file
27
internal/certauthority/kubecertauthority/testdata/test2.key
vendored
Normal file
@ -0,0 +1,27 @@
|
|||||||
|
-----BEGIN RSA PRIVATE KEY-----
|
||||||
|
MIIEogIBAAKCAQEAsfsLYmm20OLebGkPht34Fm5wKtLxcXZjCpgCvjYd4+n748Ge
|
||||||
|
VQWpaa09z39uaj7v7vOIACWcBXSklY/4dH8+yRfwxFwUADUehctrftDhK4EFAb4k
|
||||||
|
T4B5o45uZBhoIW/nl6NQXj4kihA52VtgQxhIjTt6XRzFAvgij8IcnensC00SXGMO
|
||||||
|
4wrgH1bw2IKx7entR/5CN7jB7VNg59ykAzSOkE8cr28IEKlf6PnZLkNRNFUiT2TJ
|
||||||
|
7eUIOZY5cf9+GP9olMzUw/+FOFuXbAhsW6/VulrSRc1mzzEHrd22pE5D6UKCEOa5
|
||||||
|
rJGbPBeXY9foiAUVyNXICCNetmqOKGz6Zjx5PQIDAQABAoIBAD06klYO7De8dKxz
|
||||||
|
EEZjgn+lCq2Q2EMiaTwxw2/QikPoMSHPcDrrsbaLROJngoLGmCBqY3U5ew1dbWmO
|
||||||
|
l/jr9ZuUwt2ql67il1eL/bUpAu3GewR4d2FqX25nB48j3l7ycof2RSXG1ycwIdam
|
||||||
|
2tz6M6tytMvno9c7qhguvU2ONghEreXG3YYLdf9l97aB+p6GdXhwty22b7tAVwp1
|
||||||
|
GKn79kVYgmL86lph9hBPqtHuG1LHZUiFodr2iWXSu3H/265OD58a33ZO3iyfFI0s
|
||||||
|
PPy87ZN0r+1hGpoKKkDe63udOYgAG6xmIea/1Pdn9Eg87tueoeC7XcUpdaCJlKaF
|
||||||
|
tqCusEECgYEA60rWyXxTFKJ4QdVaqXoWMA4cQkT73RxznSKwkN/Svk8TVv+p5s5Y
|
||||||
|
oYKN4qyMzxvQzu+QNWpd1yTveCmmEynz457ELpGtidtiJdm7xZMdMGrU02eCL9mZ
|
||||||
|
ERbtfAkbEAKvN8D73fWyzghKv4dgcQptmsqZlYYc4vpwHveK+/N5lukCgYEAwaT3
|
||||||
|
iMTWCv7Vp87iKrzNUAH4iBWlazwbE+EDEnHVw26Y82fhgEgxiU2UvFSaIVhGpaCz
|
||||||
|
MYSXSdRcQTHgCoJLPfWHUHTJPqf36KfAJfdaxxjzTTbNYjUOkdcUD1bcNrm0yjoY
|
||||||
|
nR4zK1FPw86ODMYtBpfkyL7ZX8G1v5pRL/6/gzUCgYBzgwQ7Wmu3H6QGPeYKecNW
|
||||||
|
yDabWh6ECKnBpPwlw5xEjbGi7lTM2NSuRde+RpPCQZebYATeFGAJdTqTNW8wzVHM
|
||||||
|
l28cpawal7dxeZkzf+u+j1P4jUJel2cL+sOQNzAwBgFbT8TWzP6BI5T+vklcdZAl
|
||||||
|
g/0uaO7Zh7Vvnnt/AaLZsQKBgGfbHzuGPjoFdPecOKatPfxUIkRyP5bk1KzzuF8T
|
||||||
|
GI/JaFTbeREBJzg5mLTtNwD9RF6ecpzzPOTG9Xet1Tgtq0cewSUAjdKB6a8pESAL
|
||||||
|
qu8vTYYzBzJNvHOxg7u6XT8omHMBd6QEx3LLGFmvFXZ6bzmjC3wzB4iY7u5FSJfS
|
||||||
|
LEqlAoGAb0rbJ85vrJopbx8lzhJjyaJfM8+A3oQg1K3TphHrcgkUc8qx8QEosziM
|
||||||
|
wzYKSBlXd2bxMibyd0mTEMNl4/BqofaKoqof9gBIbkamwXOO8s7IgKxQAfr1R/z8
|
||||||
|
tHBW/g0QWPB+qtaVDtHwyQLlxjx8HD7atIo8d/do9ruwVaf+r6g=
|
||||||
|
-----END RSA PRIVATE KEY-----
|
@ -7,9 +7,6 @@ package apicerts
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"crypto/tls"
|
|
||||||
"crypto/x509"
|
|
||||||
"encoding/pem"
|
|
||||||
"errors"
|
"errors"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
@ -222,29 +219,10 @@ func TestManagerControllerSync(t *testing.T) {
|
|||||||
r.NotEmpty(actualCertChain)
|
r.NotEmpty(actualCertChain)
|
||||||
|
|
||||||
// Validate the created cert using the CA, and also validate the cert's hostname
|
// Validate the created cert using the CA, and also validate the cert's hostname
|
||||||
roots := x509.NewCertPool()
|
validCert := testutil.ValidateCertificate(t, actualCACert, actualCertChain)
|
||||||
ok := roots.AppendCertsFromPEM([]byte(actualCACert))
|
validCert.RequireDNSName("placeholder-name-api." + installedInNamespace + ".svc")
|
||||||
r.True(ok)
|
validCert.RequireLifetime(time.Now(), time.Now().Add(24*365*time.Hour), 2*time.Minute)
|
||||||
block, _ := pem.Decode([]byte(actualCertChain))
|
validCert.RequireMatchesPrivateKey(actualPrivateKey)
|
||||||
r.NotNil(block)
|
|
||||||
parsedCert, err := x509.ParseCertificate(block.Bytes)
|
|
||||||
r.NoError(err)
|
|
||||||
serviceEndpoint := "placeholder-name-api." + installedInNamespace + ".svc"
|
|
||||||
opts := x509.VerifyOptions{
|
|
||||||
DNSName: serviceEndpoint,
|
|
||||||
Roots: roots,
|
|
||||||
}
|
|
||||||
_, err = parsedCert.Verify(opts)
|
|
||||||
r.NoError(err)
|
|
||||||
r.Contains(parsedCert.DNSNames, serviceEndpoint, "expected an explicit DNS SAN, not just Common Name")
|
|
||||||
|
|
||||||
// Check the created cert's validity bounds
|
|
||||||
r.WithinDuration(time.Now(), parsedCert.NotBefore, time.Minute*2)
|
|
||||||
r.WithinDuration(time.Now().Add(24*365*time.Hour), parsedCert.NotAfter, time.Minute*2)
|
|
||||||
|
|
||||||
// Check that the private key and cert chain match
|
|
||||||
_, err = tls.X509KeyPair([]byte(actualCertChain), []byte(actualPrivateKey))
|
|
||||||
r.NoError(err)
|
|
||||||
|
|
||||||
// Make sure we updated the APIService caBundle and left it otherwise unchanged
|
// Make sure we updated the APIService caBundle and left it otherwise unchanged
|
||||||
r.Len(aggregatorAPIClient.Actions(), 2)
|
r.Len(aggregatorAPIClient.Actions(), 2)
|
||||||
|
@ -97,7 +97,7 @@ func createClients() (
|
|||||||
placeholderClient *placeholderclientset.Clientset,
|
placeholderClient *placeholderclientset.Clientset,
|
||||||
err error,
|
err error,
|
||||||
) {
|
) {
|
||||||
// Load the Kubernetes client configuration (kubeconfig),
|
// Load the Kubernetes client configuration.
|
||||||
kubeConfig, err := restclient.InClusterConfig()
|
kubeConfig, err := restclient.InClusterConfig()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, nil, fmt.Errorf("could not load in-cluster configuration: %w", err)
|
return nil, nil, nil, fmt.Errorf("could not load in-cluster configuration: %w", err)
|
||||||
|
@ -425,9 +425,10 @@ func TestCreate(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func requireOneLogStatement(r *require.Assertions, logger *testutil.TranscriptLogger, messageContains string) {
|
func requireOneLogStatement(r *require.Assertions, logger *testutil.TranscriptLogger, messageContains string) {
|
||||||
r.Len(logger.Transcript, 1)
|
transcript := logger.Transcript()
|
||||||
r.Equal("info", logger.Transcript[0].Level)
|
r.Len(transcript, 1)
|
||||||
r.Contains(logger.Transcript[0].Message, messageContains)
|
r.Equal("info", transcript[0].Level)
|
||||||
|
r.Contains(transcript[0].Message, messageContains)
|
||||||
}
|
}
|
||||||
|
|
||||||
func callCreate(ctx context.Context, storage *REST, credentialRequest *placeholderapi.CredentialRequest) (runtime.Object, error) {
|
func callCreate(ctx context.Context, storage *REST, credentialRequest *placeholderapi.CredentialRequest) (runtime.Object, error) {
|
||||||
|
@ -10,17 +10,21 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
|
"time"
|
||||||
|
|
||||||
"github.com/spf13/cobra"
|
"github.com/spf13/cobra"
|
||||||
genericapiserver "k8s.io/apiserver/pkg/server"
|
genericapiserver "k8s.io/apiserver/pkg/server"
|
||||||
genericoptions "k8s.io/apiserver/pkg/server/options"
|
genericoptions "k8s.io/apiserver/pkg/server/options"
|
||||||
"k8s.io/apiserver/plugin/pkg/authenticator/token/webhook"
|
"k8s.io/apiserver/plugin/pkg/authenticator/token/webhook"
|
||||||
|
"k8s.io/client-go/kubernetes"
|
||||||
|
restclient "k8s.io/client-go/rest"
|
||||||
|
|
||||||
"github.com/suzerain-io/placeholder-name/internal/apiserver"
|
"github.com/suzerain-io/placeholder-name/internal/apiserver"
|
||||||
"github.com/suzerain-io/placeholder-name/internal/certauthority"
|
"github.com/suzerain-io/placeholder-name/internal/certauthority/kubecertauthority"
|
||||||
"github.com/suzerain-io/placeholder-name/internal/controllermanager"
|
"github.com/suzerain-io/placeholder-name/internal/controllermanager"
|
||||||
"github.com/suzerain-io/placeholder-name/internal/downward"
|
"github.com/suzerain-io/placeholder-name/internal/downward"
|
||||||
"github.com/suzerain-io/placeholder-name/internal/provider"
|
"github.com/suzerain-io/placeholder-name/internal/provider"
|
||||||
|
"github.com/suzerain-io/placeholder-name/internal/registry/credentialrequest"
|
||||||
placeholderv1alpha1 "github.com/suzerain-io/placeholder-name/kubernetes/1.19/api/apis/placeholder/v1alpha1"
|
placeholderv1alpha1 "github.com/suzerain-io/placeholder-name/kubernetes/1.19/api/apis/placeholder/v1alpha1"
|
||||||
"github.com/suzerain-io/placeholder-name/pkg/config"
|
"github.com/suzerain-io/placeholder-name/pkg/config"
|
||||||
)
|
)
|
||||||
@ -32,8 +36,6 @@ type App struct {
|
|||||||
// CLI flags
|
// CLI flags
|
||||||
configPath string
|
configPath string
|
||||||
downwardAPIPath string
|
downwardAPIPath string
|
||||||
clusterSigningCertFilePath string
|
|
||||||
clusterSigningKeyFilePath string
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// This is ignored for now because we turn off etcd storage below, but this is
|
// This is ignored for now because we turn off etcd storage below, but this is
|
||||||
@ -87,20 +89,6 @@ func addCommandlineFlagsToCommand(cmd *cobra.Command, app *App) {
|
|||||||
"/etc/podinfo",
|
"/etc/podinfo",
|
||||||
"path to Downward API volume mount",
|
"path to Downward API volume mount",
|
||||||
)
|
)
|
||||||
|
|
||||||
cmd.Flags().StringVar(
|
|
||||||
&app.clusterSigningCertFilePath,
|
|
||||||
"cluster-signing-cert-file",
|
|
||||||
"",
|
|
||||||
"path to cluster signing certificate",
|
|
||||||
)
|
|
||||||
|
|
||||||
cmd.Flags().StringVar(
|
|
||||||
&app.clusterSigningKeyFilePath,
|
|
||||||
"cluster-signing-key-file",
|
|
||||||
"",
|
|
||||||
"path to cluster signing private key",
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Boot the aggregated API server, which will in turn boot the controllers.
|
// Boot the aggregated API server, which will in turn boot the controllers.
|
||||||
@ -112,10 +100,11 @@ func (a *App) runServer(ctx context.Context) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Load the Kubernetes cluster signing CA.
|
// Load the Kubernetes cluster signing CA.
|
||||||
k8sClusterCA, err := certauthority.Load(a.clusterSigningCertFilePath, a.clusterSigningKeyFilePath)
|
k8sClusterCA, shutdownCA, err := getClusterCASigner()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("could not load cluster signing CA: %w", err)
|
return err
|
||||||
}
|
}
|
||||||
|
defer shutdownCA()
|
||||||
|
|
||||||
// Create a WebhookTokenAuthenticator.
|
// Create a WebhookTokenAuthenticator.
|
||||||
webhookTokenAuthenticator, err := config.NewWebhook(cfg.WebhookConfig)
|
webhookTokenAuthenticator, err := config.NewWebhook(cfg.WebhookConfig)
|
||||||
@ -169,11 +158,40 @@ func (a *App) runServer(ctx context.Context) error {
|
|||||||
return server.GenericAPIServer.PrepareRun().Run(ctx.Done())
|
return server.GenericAPIServer.PrepareRun().Run(ctx.Done())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func getClusterCASigner() (*kubecertauthority.CA, kubecertauthority.ShutdownFunc, error) {
|
||||||
|
// Load the Kubernetes client configuration.
|
||||||
|
kubeConfig, err := restclient.InClusterConfig()
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, fmt.Errorf("could not load in-cluster configuration: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Connect to the core Kubernetes API.
|
||||||
|
kubeClient, err := kubernetes.NewForConfig(kubeConfig)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, fmt.Errorf("could not initialize Kubernetes client: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Make a clock tick that triggers a periodic refresh.
|
||||||
|
ticker := time.NewTicker(5 * time.Minute)
|
||||||
|
|
||||||
|
// Make a CA which uses the Kubernetes cluster API server's signing certs.
|
||||||
|
k8sClusterCA, shutdownCA, err := kubecertauthority.New(
|
||||||
|
kubeClient,
|
||||||
|
kubecertauthority.NewPodCommandExecutor(kubeConfig, kubeClient),
|
||||||
|
ticker.C,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
ticker.Stop()
|
||||||
|
return nil, nil, fmt.Errorf("could not load cluster signing CA: %w", err)
|
||||||
|
}
|
||||||
|
return k8sClusterCA, func() { shutdownCA(); ticker.Stop() }, nil
|
||||||
|
}
|
||||||
|
|
||||||
// Create a configuration for the aggregated API server.
|
// Create a configuration for the aggregated API server.
|
||||||
func getAggregatedAPIServerConfig(
|
func getAggregatedAPIServerConfig(
|
||||||
dynamicCertProvider provider.DynamicTLSServingCertProvider,
|
dynamicCertProvider provider.DynamicTLSServingCertProvider,
|
||||||
webhookTokenAuthenticator *webhook.WebhookTokenAuthenticator,
|
webhookTokenAuthenticator *webhook.WebhookTokenAuthenticator,
|
||||||
ca *certauthority.CA,
|
ca credentialrequest.CertIssuer,
|
||||||
startControllersPostStartHook func(context.Context),
|
startControllersPostStartHook func(context.Context),
|
||||||
) (*apiserver.Config, error) {
|
) (*apiserver.Config, error) {
|
||||||
recommendedOptions := genericoptions.NewRecommendedOptions(
|
recommendedOptions := genericoptions.NewRecommendedOptions(
|
||||||
|
@ -25,8 +25,6 @@ Usage:
|
|||||||
placeholder-name-server [flags]
|
placeholder-name-server [flags]
|
||||||
|
|
||||||
Flags:
|
Flags:
|
||||||
--cluster-signing-cert-file string path to cluster signing certificate
|
|
||||||
--cluster-signing-key-file string path to cluster signing private key
|
|
||||||
-c, --config string path to configuration file (default "placeholder-name.yaml")
|
-c, --config string path to configuration file (default "placeholder-name.yaml")
|
||||||
--downward-api-path string path to Downward API volume mount (default "/etc/podinfo")
|
--downward-api-path string path to Downward API volume mount (default "/etc/podinfo")
|
||||||
-h, --help help for placeholder-name-server
|
-h, --help help for placeholder-name-server
|
||||||
|
73
internal/testutil/certs.go
Normal file
73
internal/testutil/certs.go
Normal file
@ -0,0 +1,73 @@
|
|||||||
|
/*
|
||||||
|
Copyright 2020 VMware, Inc.
|
||||||
|
SPDX-License-Identifier: Apache-2.0
|
||||||
|
*/
|
||||||
|
|
||||||
|
package testutil
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/tls"
|
||||||
|
"crypto/x509"
|
||||||
|
"encoding/pem"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
type ValidCert struct {
|
||||||
|
t *testing.T
|
||||||
|
roots *x509.CertPool
|
||||||
|
certPEM string
|
||||||
|
parsed *x509.Certificate
|
||||||
|
}
|
||||||
|
|
||||||
|
// ValidateCertificate validates a certificate and provides an object for asserting properties of the certificate.
|
||||||
|
func ValidateCertificate(t *testing.T, caPEM string, certPEM string) *ValidCert {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
block, _ := pem.Decode([]byte(certPEM))
|
||||||
|
require.NotNil(t, block)
|
||||||
|
parsed, err := x509.ParseCertificate(block.Bytes)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
// Validate the created cert using the CA.
|
||||||
|
roots := x509.NewCertPool()
|
||||||
|
require.True(t, roots.AppendCertsFromPEM([]byte(caPEM)))
|
||||||
|
opts := x509.VerifyOptions{Roots: roots}
|
||||||
|
_, err = parsed.Verify(opts)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
return &ValidCert{
|
||||||
|
t: t,
|
||||||
|
roots: roots,
|
||||||
|
certPEM: certPEM,
|
||||||
|
parsed: parsed,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// RequireDNSName asserts that the certificate matches the provided DNS name.
|
||||||
|
func (v *ValidCert) RequireDNSName(expectDNSName string) {
|
||||||
|
v.t.Helper()
|
||||||
|
opts := x509.VerifyOptions{
|
||||||
|
Roots: v.roots,
|
||||||
|
DNSName: expectDNSName,
|
||||||
|
}
|
||||||
|
_, err := v.parsed.Verify(opts)
|
||||||
|
require.NoError(v.t, err)
|
||||||
|
require.Contains(v.t, v.parsed.DNSNames, expectDNSName, "expected an explicit DNS SAN, not just Common Name")
|
||||||
|
}
|
||||||
|
|
||||||
|
// RequireLifetime asserts that the lifetime of the certificate matches the expected timestamps.
|
||||||
|
func (v *ValidCert) RequireLifetime(expectNotBefore time.Time, expectNotAfter time.Time, delta time.Duration) {
|
||||||
|
v.t.Helper()
|
||||||
|
require.WithinDuration(v.t, expectNotBefore, v.parsed.NotBefore, delta)
|
||||||
|
require.WithinDuration(v.t, expectNotAfter, v.parsed.NotAfter, delta)
|
||||||
|
}
|
||||||
|
|
||||||
|
// RequireMatchesPrivateKey asserts that the public key in the certificate matches the provided private key.
|
||||||
|
func (v *ValidCert) RequireMatchesPrivateKey(keyPEM string) {
|
||||||
|
v.t.Helper()
|
||||||
|
_, err := tls.X509KeyPair([]byte(v.certPEM), []byte(keyPEM))
|
||||||
|
require.NoError(v.t, err)
|
||||||
|
}
|
@ -7,6 +7,7 @@ package testutil
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"sync"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/go-logr/logr"
|
"github.com/go-logr/logr"
|
||||||
@ -14,7 +15,8 @@ import (
|
|||||||
|
|
||||||
type TranscriptLogger struct {
|
type TranscriptLogger struct {
|
||||||
t *testing.T
|
t *testing.T
|
||||||
Transcript []TranscriptLogMessage
|
lock sync.Mutex
|
||||||
|
transcript []TranscriptLogMessage
|
||||||
}
|
}
|
||||||
|
|
||||||
var _ logr.Logger = &TranscriptLogger{}
|
var _ logr.Logger = &TranscriptLogger{}
|
||||||
@ -28,15 +30,27 @@ func NewTranscriptLogger(t *testing.T) *TranscriptLogger {
|
|||||||
return &TranscriptLogger{t: t}
|
return &TranscriptLogger{t: t}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (log *TranscriptLogger) Transcript() []TranscriptLogMessage {
|
||||||
|
log.lock.Lock()
|
||||||
|
defer log.lock.Unlock()
|
||||||
|
result := make([]TranscriptLogMessage, 0, len(log.transcript))
|
||||||
|
result = append(result, log.transcript...)
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
func (log *TranscriptLogger) Info(msg string, keysAndValues ...interface{}) {
|
func (log *TranscriptLogger) Info(msg string, keysAndValues ...interface{}) {
|
||||||
log.Transcript = append(log.Transcript, TranscriptLogMessage{
|
log.lock.Lock()
|
||||||
|
defer log.lock.Unlock()
|
||||||
|
log.transcript = append(log.transcript, TranscriptLogMessage{
|
||||||
Level: "info",
|
Level: "info",
|
||||||
Message: fmt.Sprintf(msg, keysAndValues...),
|
Message: fmt.Sprintf(msg, keysAndValues...),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func (log *TranscriptLogger) Error(err error, msg string, keysAndValues ...interface{}) {
|
func (log *TranscriptLogger) Error(err error, msg string, keysAndValues ...interface{}) {
|
||||||
log.Transcript = append(log.Transcript, TranscriptLogMessage{
|
log.lock.Lock()
|
||||||
|
defer log.lock.Unlock()
|
||||||
|
log.transcript = append(log.transcript, TranscriptLogMessage{
|
||||||
Level: "error",
|
Level: "error",
|
||||||
Message: fmt.Sprintf("%s: %v -- %v", msg, err, keysAndValues),
|
Message: fmt.Sprintf("%s: %v -- %v", msg, err, keysAndValues),
|
||||||
})
|
})
|
||||||
|
@ -11,23 +11,39 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
|
appsv1 "k8s.io/api/apps/v1"
|
||||||
|
corev1 "k8s.io/api/core/v1"
|
||||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||||
|
|
||||||
"github.com/suzerain-io/placeholder-name/test/library"
|
"github.com/suzerain-io/placeholder-name/test/library"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestAppAvailability(t *testing.T) {
|
func TestGetDeployment(t *testing.T) {
|
||||||
library.SkipUnlessIntegration(t)
|
library.SkipUnlessIntegration(t)
|
||||||
namespaceName := library.Getenv(t, "PLACEHOLDER_NAME_NAMESPACE")
|
namespaceName := library.Getenv(t, "PLACEHOLDER_NAME_NAMESPACE")
|
||||||
daemonSetName := library.Getenv(t, "PLACEHOLDER_NAME_APP_NAME")
|
deploymentName := library.Getenv(t, "PLACEHOLDER_NAME_APP_NAME")
|
||||||
|
|
||||||
client := library.NewClientset(t)
|
client := library.NewClientset(t)
|
||||||
|
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
|
|
||||||
daemonSet, err := client.AppsV1().DaemonSets(namespaceName).Get(ctx, daemonSetName, metav1.GetOptions{})
|
appDeployment, err := client.AppsV1().Deployments(namespaceName).Get(ctx, deploymentName, metav1.GetOptions{})
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
require.GreaterOrEqual(t, daemonSet.Status.NumberAvailable, int32(1))
|
cond := getDeploymentCondition(appDeployment.Status, appsv1.DeploymentAvailable)
|
||||||
|
require.NotNil(t, cond)
|
||||||
|
require.Equalf(t, corev1.ConditionTrue, cond.Status, "app should be available: %s", library.Sdump(appDeployment))
|
||||||
|
}
|
||||||
|
|
||||||
|
// getDeploymentCondition returns the condition with the provided type.
|
||||||
|
// Copied from k8s.io/kubectl/pkg/util/deployment/deployment.go to prevent us from vendoring the world.
|
||||||
|
func getDeploymentCondition(status appsv1.DeploymentStatus, condType appsv1.DeploymentConditionType) *appsv1.DeploymentCondition {
|
||||||
|
for i := range status.Conditions {
|
||||||
|
c := status.Conditions[i]
|
||||||
|
if c.Type == condType {
|
||||||
|
return &c
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user