ContainerImage.Pinniped/internal/kubeclient/kubeclient_test.go

1165 lines
38 KiB
Go

// Copyright 2021-2022 the Pinniped contributors. All Rights Reserved.
// SPDX-License-Identifier: Apache-2.0
package kubeclient
import (
"context"
"fmt"
"io"
"net/http"
"net/url"
"runtime"
"strings"
"sync"
"testing"
"time"
"github.com/stretchr/testify/require"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/net"
clientauthenticationv1 "k8s.io/client-go/pkg/apis/clientauthentication/v1"
"k8s.io/client-go/rest"
clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
"k8s.io/client-go/transport"
apiregistrationv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1"
// register all client-go auth plugins.
_ "k8s.io/client-go/plugin/pkg/client/auth"
conciergeconfigv1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/config/v1alpha1"
supervisorconfigv1alpha1 "go.pinniped.dev/generated/latest/apis/supervisor/config/v1alpha1"
"go.pinniped.dev/internal/crypto/ptls"
"go.pinniped.dev/internal/httputil/roundtripper"
"go.pinniped.dev/internal/testutil/fakekubeapi"
)
const (
someUID = "some fake UID"
)
var (
podGVK = corev1.SchemeGroupVersion.WithKind("Pod")
goodPod = &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "good-pod",
Namespace: "good-namespace",
},
}
apiServiceGVK = apiregistrationv1.SchemeGroupVersion.WithKind("APIService")
goodAPIService = &apiregistrationv1.APIService{
ObjectMeta: metav1.ObjectMeta{
Name: "good-api-service",
},
}
credentialIssuerGVK = conciergeconfigv1alpha1.SchemeGroupVersion.WithKind("CredentialIssuer")
goodCredentialIssuer = &conciergeconfigv1alpha1.CredentialIssuer{
ObjectMeta: metav1.ObjectMeta{
Name: "good-credential-issuer",
},
}
federationDomainGVK = supervisorconfigv1alpha1.SchemeGroupVersion.WithKind("FederationDomain")
goodFederationDomain = &supervisorconfigv1alpha1.FederationDomain{
ObjectMeta: metav1.ObjectMeta{
Name: "good-federation-domain",
Namespace: "good-namespace",
},
}
middlewareAnnotations = map[string]string{"some-annotation": "thing 1"}
middlewareLabels = map[string]string{"some-label": "thing 2"}
)
func TestKubeclient(t *testing.T) {
// plog.ValidateAndSetLogLevelGlobally(plog.LevelDebug) // uncomment me to get some more debug logs
tests := []struct {
name string
editRestConfig func(t *testing.T, restConfig *rest.Config)
middlewares func(t *testing.T) []*spyMiddleware
reallyRunTest func(t *testing.T, c *Client)
wantMiddlewareReqs, wantMiddlewareResps [][]Object
}{
{
name: "crud core api",
middlewares: func(t *testing.T) []*spyMiddleware {
return []*spyMiddleware{newAnnotationMiddleware(t), newLabelMiddleware(t)}
},
reallyRunTest: func(t *testing.T, c *Client) {
// create
pod, err := c.Kubernetes.
CoreV1().
Pods(goodPod.Namespace).
Create(context.Background(), goodPod, metav1.CreateOptions{})
require.NoError(t, err)
require.Equal(t, goodPod, pod)
// read
pod, err = c.Kubernetes.
CoreV1().
Pods(pod.Namespace).
Get(context.Background(), pod.Name, metav1.GetOptions{})
require.NoError(t, err)
require.Equal(t, with(goodPod, annotations(), labels()), pod)
// read when not found
_, err = c.Kubernetes.
CoreV1().
Pods(pod.Namespace).
Get(context.Background(), "this-pod-does-not-exist", metav1.GetOptions{})
require.EqualError(t, err, `couldn't find object for path "/api/v1/namespaces/good-namespace/pods/this-pod-does-not-exist"`)
// update
goodPodWithAnnotationsAndLabelsAndUID := with(goodPod, annotations(), labels(), uid()).(*corev1.Pod)
pod, err = c.Kubernetes.
CoreV1().
Pods(pod.Namespace).
Update(context.Background(), goodPodWithAnnotationsAndLabelsAndUID, metav1.UpdateOptions{})
require.NoError(t, err)
require.Equal(t, goodPodWithAnnotationsAndLabelsAndUID, pod)
// delete
err = c.Kubernetes.
CoreV1().
Pods(pod.Namespace).
Delete(context.Background(), pod.Name, metav1.DeleteOptions{})
require.NoError(t, err)
},
wantMiddlewareReqs: [][]Object{
{
with(goodPod, gvk(podGVK)),
with(&metav1.PartialObjectMetadata{}, gvk(podGVK)),
with(&metav1.PartialObjectMetadata{}, gvk(podGVK)),
with(goodPod, annotations(), labels(), uid(), gvk(podGVK)),
with(&metav1.PartialObjectMetadata{}, gvk(podGVK)),
},
{
with(goodPod, annotations(), gvk(podGVK)),
with(&metav1.PartialObjectMetadata{}, gvk(podGVK)),
with(&metav1.PartialObjectMetadata{}, gvk(podGVK)),
with(goodPod, annotations(), labels(), uid(), gvk(podGVK)),
with(&metav1.PartialObjectMetadata{}, gvk(podGVK)),
},
},
wantMiddlewareResps: [][]Object{
{
with(goodPod, annotations(), labels(), gvk(podGVK)),
with(goodPod, annotations(), labels(), gvk(podGVK)),
with(goodPod, annotations(), labels(), uid(), gvk(podGVK)),
},
{
with(goodPod, emptyAnnotations(), labels(), gvk(podGVK)),
with(goodPod, annotations(), labels(), gvk(podGVK)),
with(goodPod, annotations(), labels(), uid(), gvk(podGVK)),
},
},
},
{
name: "crud core api without middlewares",
reallyRunTest: func(t *testing.T, c *Client) {
// create
pod, err := c.Kubernetes.
CoreV1().
Pods(goodPod.Namespace).
Create(context.Background(), goodPod, metav1.CreateOptions{})
require.NoError(t, err)
require.Equal(t, goodPod, pod)
// read
pod, err = c.Kubernetes.
CoreV1().
Pods(pod.Namespace).
Get(context.Background(), pod.Name, metav1.GetOptions{})
require.NoError(t, err)
require.Equal(t, with(goodPod), pod)
// update
pod, err = c.Kubernetes.
CoreV1().
Pods(pod.Namespace).
Update(context.Background(), goodPod, metav1.UpdateOptions{})
require.NoError(t, err)
require.Equal(t, goodPod, pod)
// delete
err = c.Kubernetes.
CoreV1().
Pods(pod.Namespace).
Delete(context.Background(), pod.Name, metav1.DeleteOptions{})
require.NoError(t, err)
},
},
{
name: "crud aggregation api",
middlewares: func(t *testing.T) []*spyMiddleware {
return []*spyMiddleware{newAnnotationMiddleware(t), newLabelMiddleware(t)}
},
reallyRunTest: func(t *testing.T, c *Client) {
// create
apiService, err := c.Aggregation.
ApiregistrationV1().
APIServices().
Create(context.Background(), goodAPIService, metav1.CreateOptions{})
require.NoError(t, err)
require.Equal(t, goodAPIService, apiService)
// read
apiService, err = c.Aggregation.
ApiregistrationV1().
APIServices().
Get(context.Background(), apiService.Name, metav1.GetOptions{})
require.NoError(t, err)
require.Equal(t, with(goodAPIService, annotations(), labels()), apiService)
// update
goodAPIServiceWithAnnotationsAndLabelsAndUID := with(goodAPIService, annotations(), labels(), uid()).(*apiregistrationv1.APIService)
apiService, err = c.Aggregation.
ApiregistrationV1().
APIServices().
Update(context.Background(), goodAPIServiceWithAnnotationsAndLabelsAndUID, metav1.UpdateOptions{})
require.NoError(t, err)
require.Equal(t, goodAPIServiceWithAnnotationsAndLabelsAndUID, apiService)
// delete
err = c.Aggregation.
ApiregistrationV1().
APIServices().
Delete(context.Background(), apiService.Name, metav1.DeleteOptions{})
require.NoError(t, err)
},
wantMiddlewareReqs: [][]Object{
{
with(goodAPIService, gvk(apiServiceGVK)),
with(&metav1.PartialObjectMetadata{}, gvk(apiServiceGVK)),
with(goodAPIService, annotations(), labels(), uid(), gvk(apiServiceGVK)),
with(&metav1.PartialObjectMetadata{}, gvk(apiServiceGVK)),
},
{
with(goodAPIService, annotations(), gvk(apiServiceGVK)),
with(&metav1.PartialObjectMetadata{}, gvk(apiServiceGVK)),
with(goodAPIService, annotations(), labels(), uid(), gvk(apiServiceGVK)),
with(&metav1.PartialObjectMetadata{}, gvk(apiServiceGVK)),
},
},
wantMiddlewareResps: [][]Object{
{
with(goodAPIService, annotations(), labels(), gvk(apiServiceGVK)),
with(goodAPIService, annotations(), labels(), gvk(apiServiceGVK)),
with(goodAPIService, annotations(), labels(), uid(), gvk(apiServiceGVK)),
},
{
with(goodAPIService, emptyAnnotations(), labels(), gvk(apiServiceGVK)),
with(goodAPIService, annotations(), labels(), gvk(apiServiceGVK)),
with(goodAPIService, annotations(), labels(), uid(), gvk(apiServiceGVK)),
},
},
},
{
name: "crud concierge api",
middlewares: func(t *testing.T) []*spyMiddleware {
return []*spyMiddleware{newAnnotationMiddleware(t), newLabelMiddleware(t)}
},
reallyRunTest: func(t *testing.T, c *Client) {
// create
tokenCredentialRequest, err := c.PinnipedConcierge.
ConfigV1alpha1().
CredentialIssuers().
Create(context.Background(), goodCredentialIssuer, metav1.CreateOptions{})
require.NoError(t, err)
require.Equal(t, goodCredentialIssuer, tokenCredentialRequest)
// read
tokenCredentialRequest, err = c.PinnipedConcierge.
ConfigV1alpha1().
CredentialIssuers().
Get(context.Background(), tokenCredentialRequest.Name, metav1.GetOptions{})
require.NoError(t, err)
require.Equal(t, with(goodCredentialIssuer, annotations(), labels()), tokenCredentialRequest)
// update
goodCredentialIssuerWithAnnotationsAndLabelsAndUID := with(goodCredentialIssuer, annotations(), labels(), uid()).(*conciergeconfigv1alpha1.CredentialIssuer)
tokenCredentialRequest, err = c.PinnipedConcierge.
ConfigV1alpha1().
CredentialIssuers().
Update(context.Background(), goodCredentialIssuerWithAnnotationsAndLabelsAndUID, metav1.UpdateOptions{})
require.NoError(t, err)
require.Equal(t, goodCredentialIssuerWithAnnotationsAndLabelsAndUID, tokenCredentialRequest)
// delete
err = c.PinnipedConcierge.
ConfigV1alpha1().
CredentialIssuers().
Delete(context.Background(), tokenCredentialRequest.Name, metav1.DeleteOptions{})
require.NoError(t, err)
},
wantMiddlewareReqs: [][]Object{
{
with(goodCredentialIssuer, gvk(credentialIssuerGVK)),
with(&metav1.PartialObjectMetadata{}, gvk(credentialIssuerGVK)),
with(goodCredentialIssuer, annotations(), labels(), uid(), gvk(credentialIssuerGVK)),
with(&metav1.PartialObjectMetadata{}, gvk(credentialIssuerGVK)),
},
{
with(goodCredentialIssuer, annotations(), gvk(credentialIssuerGVK)),
with(&metav1.PartialObjectMetadata{}, gvk(credentialIssuerGVK)),
with(goodCredentialIssuer, annotations(), labels(), uid(), gvk(credentialIssuerGVK)),
with(&metav1.PartialObjectMetadata{}, gvk(credentialIssuerGVK)),
},
},
wantMiddlewareResps: [][]Object{
{
with(goodCredentialIssuer, annotations(), labels(), gvk(credentialIssuerGVK)),
with(goodCredentialIssuer, annotations(), labels(), gvk(credentialIssuerGVK)),
with(goodCredentialIssuer, annotations(), labels(), uid(), gvk(credentialIssuerGVK)),
},
{
with(goodCredentialIssuer, emptyAnnotations(), labels(), gvk(credentialIssuerGVK)),
with(goodCredentialIssuer, annotations(), labels(), gvk(credentialIssuerGVK)),
with(goodCredentialIssuer, annotations(), labels(), uid(), gvk(credentialIssuerGVK)),
},
},
},
{
name: "crud supervisor api",
middlewares: func(t *testing.T) []*spyMiddleware {
return []*spyMiddleware{newAnnotationMiddleware(t), newLabelMiddleware(t)}
},
reallyRunTest: func(t *testing.T, c *Client) {
// create
federationDomain, err := c.PinnipedSupervisor.
ConfigV1alpha1().
FederationDomains(goodFederationDomain.Namespace).
Create(context.Background(), goodFederationDomain, metav1.CreateOptions{})
require.NoError(t, err)
require.Equal(t, goodFederationDomain, federationDomain)
// read
federationDomain, err = c.PinnipedSupervisor.
ConfigV1alpha1().
FederationDomains(federationDomain.Namespace).
Get(context.Background(), federationDomain.Name, metav1.GetOptions{})
require.NoError(t, err)
require.Equal(t, with(goodFederationDomain, annotations(), labels()), federationDomain)
// update
goodFederationDomainWithAnnotationsAndLabelsAndUID := with(goodFederationDomain, annotations(), labels(), uid()).(*supervisorconfigv1alpha1.FederationDomain)
federationDomain, err = c.PinnipedSupervisor.
ConfigV1alpha1().
FederationDomains(federationDomain.Namespace).
Update(context.Background(), goodFederationDomainWithAnnotationsAndLabelsAndUID, metav1.UpdateOptions{})
require.NoError(t, err)
require.Equal(t, goodFederationDomainWithAnnotationsAndLabelsAndUID, federationDomain)
// delete
err = c.PinnipedSupervisor.
ConfigV1alpha1().
FederationDomains(federationDomain.Namespace).
Delete(context.Background(), federationDomain.Name, metav1.DeleteOptions{})
require.NoError(t, err)
},
wantMiddlewareReqs: [][]Object{
{
with(goodFederationDomain, gvk(federationDomainGVK)),
with(&metav1.PartialObjectMetadata{}, gvk(federationDomainGVK)),
with(goodFederationDomain, annotations(), labels(), uid(), gvk(federationDomainGVK)),
with(&metav1.PartialObjectMetadata{}, gvk(federationDomainGVK)),
},
{
with(goodFederationDomain, annotations(), gvk(federationDomainGVK)),
with(&metav1.PartialObjectMetadata{}, gvk(federationDomainGVK)),
with(goodFederationDomain, annotations(), labels(), uid(), gvk(federationDomainGVK)),
with(&metav1.PartialObjectMetadata{}, gvk(federationDomainGVK)),
},
},
wantMiddlewareResps: [][]Object{
{
with(goodFederationDomain, annotations(), labels(), gvk(federationDomainGVK)),
with(goodFederationDomain, annotations(), labels(), gvk(federationDomainGVK)),
with(goodFederationDomain, annotations(), labels(), uid(), gvk(federationDomainGVK)),
},
{
with(goodFederationDomain, emptyAnnotations(), labels(), gvk(federationDomainGVK)),
with(goodFederationDomain, annotations(), labels(), gvk(federationDomainGVK)),
with(goodFederationDomain, annotations(), labels(), uid(), gvk(federationDomainGVK)),
},
},
},
{
name: "we don't call any middleware if there are no mutation funcs",
middlewares: func(t *testing.T) []*spyMiddleware {
return []*spyMiddleware{newSimpleMiddleware(t, false, false, false), newSimpleMiddleware(t, false, false, false)}
},
reallyRunTest: createGetFederationDomainTest,
wantMiddlewareReqs: [][]Object{nil, nil},
wantMiddlewareResps: [][]Object{nil, nil},
},
{
name: "we don't call any resp middleware if there was no req mutations done and there are no resp mutation funcs",
middlewares: func(t *testing.T) []*spyMiddleware {
return []*spyMiddleware{newSimpleMiddleware(t, true, false, false), newSimpleMiddleware(t, true, false, false)}
},
reallyRunTest: createGetFederationDomainTest,
wantMiddlewareReqs: [][]Object{
{
with(goodFederationDomain, gvk(federationDomainGVK)),
with(&metav1.PartialObjectMetadata{}, gvk(federationDomainGVK)),
},
{
with(goodFederationDomain, gvk(federationDomainGVK)),
with(&metav1.PartialObjectMetadata{}, gvk(federationDomainGVK)),
},
},
wantMiddlewareResps: [][]Object{nil, nil},
},
{
name: "we don't call any resp middleware if there are no resp mutation funcs even if there was req mutations done",
middlewares: func(t *testing.T) []*spyMiddleware {
return []*spyMiddleware{newSimpleMiddleware(t, true, true, false), newSimpleMiddleware(t, true, true, false)}
},
reallyRunTest: func(t *testing.T, c *Client) {
// create
federationDomain, err := c.PinnipedSupervisor.
ConfigV1alpha1().
FederationDomains(goodFederationDomain.Namespace).
Create(context.Background(), goodFederationDomain, metav1.CreateOptions{})
require.NoError(t, err)
require.Equal(t, with(goodFederationDomain, uid()), federationDomain)
// read
federationDomain, err = c.PinnipedSupervisor.
ConfigV1alpha1().
FederationDomains(federationDomain.Namespace).
Get(context.Background(), federationDomain.Name, metav1.GetOptions{})
require.NoError(t, err)
require.Equal(t, with(goodFederationDomain, uid()), federationDomain)
},
wantMiddlewareReqs: [][]Object{
{
with(goodFederationDomain, gvk(federationDomainGVK)),
with(&metav1.PartialObjectMetadata{}, gvk(federationDomainGVK)),
},
{
with(goodFederationDomain, uid(), gvk(federationDomainGVK)),
with(&metav1.PartialObjectMetadata{}, gvk(federationDomainGVK)),
},
},
wantMiddlewareResps: [][]Object{nil, nil},
},
{
name: "we still call resp middleware if there is a resp mutation func even if there were req mutation funcs",
middlewares: func(t *testing.T) []*spyMiddleware {
return []*spyMiddleware{newSimpleMiddleware(t, false, false, true), newSimpleMiddleware(t, false, false, true)}
},
reallyRunTest: createGetFederationDomainTest,
wantMiddlewareReqs: [][]Object{nil, nil},
wantMiddlewareResps: [][]Object{
{
with(goodFederationDomain, gvk(federationDomainGVK)),
with(goodFederationDomain, gvk(federationDomainGVK)),
},
{
with(goodFederationDomain, gvk(federationDomainGVK)),
with(goodFederationDomain, gvk(federationDomainGVK)),
},
},
},
{
name: "we still call resp middleware if there is a resp mutation func even if there was no req mutation",
middlewares: func(t *testing.T) []*spyMiddleware {
return []*spyMiddleware{newSimpleMiddleware(t, true, false, true), newSimpleMiddleware(t, true, false, true)}
},
reallyRunTest: createGetFederationDomainTest,
wantMiddlewareReqs: [][]Object{
{
with(goodFederationDomain, gvk(federationDomainGVK)),
with(&metav1.PartialObjectMetadata{}, gvk(federationDomainGVK)),
},
{
with(goodFederationDomain, gvk(federationDomainGVK)),
with(&metav1.PartialObjectMetadata{}, gvk(federationDomainGVK)),
},
},
wantMiddlewareResps: [][]Object{
{
with(goodFederationDomain, gvk(federationDomainGVK)),
with(goodFederationDomain, gvk(federationDomainGVK)),
},
{
with(goodFederationDomain, gvk(federationDomainGVK)),
with(goodFederationDomain, gvk(federationDomainGVK)),
},
},
},
{
name: "mutating object meta on a get request is not allowed since that isn't pertinent to the api request",
middlewares: func(t *testing.T) []*spyMiddleware {
return []*spyMiddleware{{
name: "non-pertinent mutater",
t: t,
mutateReq: func(rt RoundTrip, obj Object) error {
uid()(obj)
return nil
},
}}
},
reallyRunTest: func(t *testing.T, c *Client) {
_, err := c.PinnipedSupervisor.
ConfigV1alpha1().
FederationDomains(goodFederationDomain.Namespace).
Get(context.Background(), goodFederationDomain.Name, metav1.GetOptions{})
require.Error(t, err)
require.Contains(t, err.Error(), "invalid object meta mutation")
},
wantMiddlewareReqs: [][]Object{{with(&metav1.PartialObjectMetadata{}, gvk(federationDomainGVK))}},
wantMiddlewareResps: [][]Object{nil},
},
{
name: "when the client gets errors from the api server",
middlewares: func(t *testing.T) []*spyMiddleware {
return []*spyMiddleware{newSimpleMiddleware(t, true, false, false)}
},
editRestConfig: func(t *testing.T, restConfig *rest.Config) {
// avoid messing with restConfig.Dial since it breaks client-go TLS cache logic
restConfig.Wrap(func(rt http.RoundTripper) http.RoundTripper {
return roundtripper.WrapFunc(rt, func(_ *http.Request) (*http.Response, error) {
return nil, fmt.Errorf("some fake connection error")
})
})
},
reallyRunTest: func(t *testing.T, c *Client) {
_, err := c.PinnipedSupervisor.
ConfigV1alpha1().
FederationDomains(goodFederationDomain.Namespace).
Get(context.Background(), goodFederationDomain.Name, metav1.GetOptions{})
require.Error(t, err)
require.Contains(t, err.Error(), ": some fake connection error")
},
wantMiddlewareReqs: [][]Object{{with(&metav1.PartialObjectMetadata{}, gvk(federationDomainGVK))}},
wantMiddlewareResps: [][]Object{nil},
},
{
name: "when there are request middleware failures, we return an error and don't send the request",
middlewares: func(t *testing.T) []*spyMiddleware {
return []*spyMiddleware{
// use 3 middleware to ensure that we collect all errors from all middlewares
newFailingMiddleware(t, "aaa", true, false),
newFailingMiddleware(t, "bbb", false, false),
newFailingMiddleware(t, "ccc", true, false),
}
},
reallyRunTest: func(t *testing.T, c *Client) {
_, err := c.PinnipedSupervisor.
ConfigV1alpha1().
FederationDomains(goodFederationDomain.Namespace).
Get(context.Background(), goodFederationDomain.Name, metav1.GetOptions{})
require.Error(t, err)
require.Contains(t, err.Error(), ": request mutation failed: [aaa: request error, ccc: request error]")
},
wantMiddlewareReqs: [][]Object{
{with(&metav1.PartialObjectMetadata{}, gvk(federationDomainGVK))},
{with(&metav1.PartialObjectMetadata{}, gvk(federationDomainGVK))},
{with(&metav1.PartialObjectMetadata{}, gvk(federationDomainGVK))},
},
wantMiddlewareResps: [][]Object{
nil,
nil,
nil,
},
},
{
name: "when there are response middleware failures, we return an error",
middlewares: func(t *testing.T) []*spyMiddleware {
return []*spyMiddleware{
// use 3 middleware to ensure that we collect all errors from all middlewares
newFailingMiddleware(t, "aaa", false, true),
newFailingMiddleware(t, "bbb", false, false),
newFailingMiddleware(t, "ccc", false, true),
}
},
reallyRunTest: func(t *testing.T, c *Client) {
_, err := c.PinnipedSupervisor.
ConfigV1alpha1().
FederationDomains(goodFederationDomain.Namespace).
Create(context.Background(), goodFederationDomain, metav1.CreateOptions{})
require.Error(t, err)
require.Contains(t, err.Error(), ": response mutation failed: [aaa: response error, ccc: response error]")
},
wantMiddlewareReqs: [][]Object{
{with(goodFederationDomain, gvk(federationDomainGVK))},
{with(goodFederationDomain, gvk(federationDomainGVK))},
{with(goodFederationDomain, gvk(federationDomainGVK))},
},
wantMiddlewareResps: [][]Object{
{with(goodFederationDomain, gvk(federationDomainGVK))},
{with(goodFederationDomain, gvk(federationDomainGVK))},
{with(goodFederationDomain, gvk(federationDomainGVK))},
},
},
}
for _, test := range tests {
test := test
t.Run(test.name, func(t *testing.T) {
_, restConfig := fakekubeapi.Start(t, nil)
if test.editRestConfig != nil {
test.editRestConfig(t, restConfig)
}
var middlewares []*spyMiddleware
if test.middlewares != nil {
middlewares = test.middlewares(t)
}
// our rt chain is:
// wantCloseReq -> kubeclient -> wantCloseResp -> http.DefaultTransport -> wantCloseResp -> kubeclient -> wantCloseReq
restConfig.Wrap(wantCloseRespWrapper(t))
opts := []Option{WithConfig(restConfig), WithTransportWrapper(wantCloseReqWrapper(t))}
for _, middleware := range middlewares {
opts = append(opts, WithMiddleware(middleware))
}
client, err := New(opts...)
require.NoError(t, err)
test.reallyRunTest(t, client)
for i, spyMiddleware := range middlewares {
require.Equalf(t, test.wantMiddlewareReqs[i], spyMiddleware.reqObjs, "unexpected req obj in middleware %q (index %d)", spyMiddleware.name, i)
require.Equalf(t, test.wantMiddlewareResps[i], spyMiddleware.respObjs, "unexpected resp obj in middleware %q (index %d)", spyMiddleware.name, i)
}
})
}
}
type spyMiddleware struct {
name string
t *testing.T
mutateReq func(RoundTrip, Object) error
mutateResp func(RoundTrip, Object) error
reqObjs []Object
respObjs []Object
}
func (s *spyMiddleware) Handle(_ context.Context, rt RoundTrip) {
s.t.Log(s.name, "handling", reqStr(rt, nil))
if s.mutateReq != nil {
rt.MutateRequest(func(obj Object) error {
s.t.Log(s.name, "mutating request", reqStr(rt, obj))
s.reqObjs = append(s.reqObjs, obj.DeepCopyObject().(Object))
return s.mutateReq(rt, obj)
})
}
if s.mutateResp != nil {
rt.MutateResponse(func(obj Object) error {
s.t.Log(s.name, "mutating response", reqStr(rt, obj))
s.respObjs = append(s.respObjs, obj.DeepCopyObject().(Object))
return s.mutateResp(rt, obj)
})
}
}
func reqStr(rt RoundTrip, obj Object) string {
b := strings.Builder{}
fmt.Fprintf(&b, "%s /%s", rt.Verb(), rt.Resource().GroupVersion())
if rt.NamespaceScoped() {
fmt.Fprintf(&b, "/namespaces/%s", rt.Namespace())
}
fmt.Fprintf(&b, "/%s", rt.Resource().Resource)
if obj != nil {
fmt.Fprintf(&b, "/%s", obj.GetName())
}
return b.String()
}
func newAnnotationMiddleware(t *testing.T) *spyMiddleware {
return &spyMiddleware{
name: "annotater",
t: t,
mutateReq: func(rt RoundTrip, obj Object) error {
if rt.Verb() == VerbCreate {
annotations()(obj)
}
return nil
},
mutateResp: func(rt RoundTrip, obj Object) error {
if rt.Verb() == VerbCreate {
for key := range middlewareAnnotations {
delete(obj.GetAnnotations(), key)
}
}
return nil
},
}
}
func newLabelMiddleware(t *testing.T) *spyMiddleware {
return &spyMiddleware{
name: "labeler",
t: t,
mutateReq: func(rt RoundTrip, obj Object) error {
if rt.Verb() == VerbCreate {
labels()(obj)
}
return nil
},
mutateResp: func(rt RoundTrip, obj Object) error {
if rt.Verb() == VerbCreate {
for key := range middlewareLabels {
delete(obj.GetLabels(), key)
}
}
return nil
},
}
}
func newSimpleMiddleware(t *testing.T, hasMutateReqFunc, mutatedReq, hasMutateRespFunc bool) *spyMiddleware {
m := &spyMiddleware{
name: "simple",
t: t,
}
if hasMutateReqFunc {
m.mutateReq = func(rt RoundTrip, obj Object) error {
if mutatedReq {
if rt.Verb() == VerbCreate {
obj.SetUID(someUID)
}
}
return nil
}
}
if hasMutateRespFunc {
m.mutateResp = func(rt RoundTrip, obj Object) error {
return nil
}
}
return m
}
func newFailingMiddleware(t *testing.T, name string, mutateReqFails, mutateRespFails bool) *spyMiddleware {
m := &spyMiddleware{
name: "failing-middleware-" + name,
t: t,
}
m.mutateReq = func(rt RoundTrip, obj Object) error {
if mutateReqFails {
return fmt.Errorf("%s: request error", name)
}
return nil
}
m.mutateResp = func(rt RoundTrip, obj Object) error {
if mutateRespFails {
return fmt.Errorf("%s: response error", name)
}
return nil
}
return m
}
type wantCloser struct {
m sync.Mutex
_rc io.ReadCloser
_closeCalls []string
_couldReadBytesJustBeforeClosing bool
}
func (w *wantCloser) Close() error {
w.m.Lock()
defer w.m.Unlock()
w._closeCalls = append(w._closeCalls, getCaller())
n, _ := w._rc.Read([]byte{0})
if n > 0 {
// there were still bytes left to be read
w._couldReadBytesJustBeforeClosing = true
}
return w._rc.Close()
}
func (w *wantCloser) Read(p []byte) (int, error) {
w.m.Lock()
defer w.m.Unlock()
return w._rc.Read(p)
}
func (w *wantCloser) couldRead() bool {
w.m.Lock()
defer w.m.Unlock()
return w._couldReadBytesJustBeforeClosing
}
func (w *wantCloser) calls() []string {
w.m.Lock()
defer w.m.Unlock()
return w._closeCalls
}
func getCaller() string {
_, file, line, ok := runtime.Caller(2)
if !ok {
file = "???"
line = 0
}
return fmt.Sprintf("%s:%d", file, line)
}
// wantCloseReqWrapper returns a transport.WrapperFunc that validates that the http.Request
// passed to the underlying http.RoundTripper is closed properly.
func wantCloseReqWrapper(t *testing.T) transport.WrapperFunc {
caller := getCaller()
return func(rt http.RoundTripper) http.RoundTripper {
return roundtripper.WrapFunc(rt, roundTripperFunc(func(req *http.Request) (bool, *http.Response, error) {
if req.Body != nil {
wc := &wantCloser{_rc: req.Body}
t.Cleanup(func() {
require.Eventuallyf(t, func() bool {
return 1 == len(wc.calls())
}, 5*time.Second, 100*time.Millisecond,
"did not close req body expected number of times at %s for req %#v; actual calls = %s", caller, req, wc.calls())
})
req.Body = wc
}
if req.GetBody != nil {
originalBodyCopy, originalErr := req.GetBody()
req.GetBody = func() (io.ReadCloser, error) {
if originalErr != nil {
return nil, originalErr
}
wc := &wantCloser{_rc: originalBodyCopy}
t.Cleanup(func() {
require.Eventuallyf(t, func() bool {
return 1 == len(wc.calls())
}, 5*time.Second, 100*time.Millisecond,
"did not close req body copy expected number of times at %s for req %#v; actual calls = %s", caller, req, wc.calls())
})
return wc, nil
}
}
resp, err := rt.RoundTrip(req)
return false, resp, err
}).RoundTrip)
}
}
// wantCloseRespWrapper returns a transport.WrapperFunc that validates that the http.Response
// returned by the underlying http.RoundTripper is closed properly.
func wantCloseRespWrapper(t *testing.T) transport.WrapperFunc {
caller := getCaller()
return func(rt http.RoundTripper) http.RoundTripper {
return roundtripper.WrapFunc(rt, roundTripperFunc(func(req *http.Request) (bool, *http.Response, error) {
resp, err := rt.RoundTrip(req)
if err != nil {
// request failed, so there is no response body to watch for Close() calls on
return false, resp, err
}
wc := &wantCloser{_rc: resp.Body}
t.Cleanup(func() {
require.Eventuallyf(t, func() bool {
return wc.couldRead() == false &&
1 == len(wc.calls())
}, 5*time.Second, 10*time.Millisecond,
`did not close resp body expected number of times at %s for req %#v; actual calls = %s
did not consume all response body bytes before closing %s, couldRead=%v`, caller, req, wc.calls(), caller, wc.couldRead())
})
resp.Body = wc
return false, resp, err
}).RoundTrip)
}
}
type withFunc func(obj Object)
func with(obj Object, withFuncs ...withFunc) Object {
obj = obj.DeepCopyObject().(Object)
for _, withFunc := range withFuncs {
withFunc(obj)
}
return obj
}
func gvk(gvk schema.GroupVersionKind) withFunc {
return func(obj Object) {
obj.GetObjectKind().SetGroupVersionKind(gvk)
}
}
func annotations() withFunc {
return func(obj Object) {
obj.SetAnnotations(middlewareAnnotations)
}
}
func emptyAnnotations() withFunc {
return func(obj Object) {
obj.SetAnnotations(make(map[string]string))
}
}
func labels() withFunc {
return func(obj Object) {
obj.SetLabels(middlewareLabels)
}
}
func uid() withFunc {
return func(obj Object) {
obj.SetUID(someUID)
}
}
func createGetFederationDomainTest(t *testing.T, client *Client) {
t.Helper()
// create
federationDomain, err := client.PinnipedSupervisor.
ConfigV1alpha1().
FederationDomains(goodFederationDomain.Namespace).
Create(context.Background(), goodFederationDomain, metav1.CreateOptions{})
require.NoError(t, err)
require.Equal(t, goodFederationDomain, federationDomain)
// read
federationDomain, err = client.PinnipedSupervisor.
ConfigV1alpha1().
FederationDomains(federationDomain.Namespace).
Get(context.Background(), federationDomain.Name, metav1.GetOptions{})
require.NoError(t, err)
require.Equal(t, goodFederationDomain, federationDomain)
}
// TestUnwrap ensures that the Client struct returned by this package only contains
// transports that can be fully unwrapped to get access to the underlying TLS config.
func TestUnwrap(t *testing.T) {
t.Parallel() // make sure to run in parallel to confirm that our client-go TLS cache busting works (i.e. assert no data races)
server, restConfig := fakekubeapi.Start(t, nil)
serverSubjects := server.Client().Transport.(*http.Transport).TLSClientConfig.RootCAs.Subjects()
t.Run("regular client", func(t *testing.T) {
t.Parallel() // make sure to run in parallel to confirm that our client-go TLS cache busting works (i.e. assert no data races)
regularClient := makeClient(t, restConfig, func(_ *rest.Config) {})
testUnwrap(t, regularClient, serverSubjects)
})
t.Run("exec client", func(t *testing.T) {
t.Parallel() // make sure to run in parallel to confirm that our client-go TLS cache busting works (i.e. assert no data races)
execClient := makeClient(t, restConfig, func(config *rest.Config) {
config.ExecProvider = &clientcmdapi.ExecConfig{
Command: "echo",
Args: []string{"pandas are awesome"},
APIVersion: clientauthenticationv1.SchemeGroupVersion.String(),
InteractiveMode: clientcmdapi.NeverExecInteractiveMode,
}
})
testUnwrap(t, execClient, serverSubjects)
})
t.Run("gcp client", func(t *testing.T) {
t.Parallel() // make sure to run in parallel to confirm that our client-go TLS cache busting works (i.e. assert no data races)
gcpClient := makeClient(t, restConfig, func(config *rest.Config) {
config.AuthProvider = &clientcmdapi.AuthProviderConfig{
Name: "gcp",
Config: map[string]string{
"cmd-path": `echo {"access_token":"fake","token_expiry":"2200-01-02T15:04:05.999999999Z07:00"}`,
},
}
})
testUnwrap(t, gcpClient, serverSubjects)
})
t.Run("oidc client", func(t *testing.T) {
t.Parallel() // make sure to run in parallel to confirm that our client-go TLS cache busting works (i.e. assert no data races)
oidcClient := makeClient(t, restConfig, func(config *rest.Config) {
config.AuthProvider = &clientcmdapi.AuthProviderConfig{
Name: "oidc",
Config: map[string]string{
"idp-issuer-url": "https://pandas.local",
"client-id": "walrus",
},
}
})
testUnwrap(t, oidcClient, serverSubjects)
})
t.Run("azure client", func(t *testing.T) {
t.Parallel() // make sure to run in parallel to confirm that our client-go TLS cache busting works (i.e. assert no data races)
azureClient := makeClient(t, restConfig, func(config *rest.Config) {
config.AuthProvider = &clientcmdapi.AuthProviderConfig{
Name: "azure",
Config: map[string]string{
"client-id": "pinny",
"tenant-id": "danger",
"apiserver-id": "1234",
},
}
})
testUnwrap(t, azureClient, serverSubjects)
})
}
func testUnwrap(t *testing.T, client *Client, serverSubjects [][]byte) {
tests := []struct {
name string
rt http.RoundTripper
}{
{
name: "core v1",
rt: extractTransport(client.Kubernetes.CoreV1()),
},
{
name: "coordination v1",
rt: extractTransport(client.Kubernetes.CoordinationV1()),
},
{
name: "api registration v1",
rt: extractTransport(client.Aggregation.ApiregistrationV1()),
},
{
name: "concierge login",
rt: extractTransport(client.PinnipedConcierge.LoginV1alpha1()),
},
{
name: "concierge config",
rt: extractTransport(client.PinnipedConcierge.ConfigV1alpha1()),
},
{
name: "supervisor idp",
rt: extractTransport(client.PinnipedSupervisor.IDPV1alpha1()),
},
{
name: "supervisor config",
rt: extractTransport(client.PinnipedSupervisor.ConfigV1alpha1()),
},
{
name: "json config",
rt: configToTransport(t, client.JSONConfig),
},
{
name: "proto config",
rt: configToTransport(t, client.ProtoConfig),
},
{
name: "anonymous json config",
rt: configToTransport(t, SecureAnonymousClientConfig(client.JSONConfig)),
},
{
name: "anonymous proto config",
rt: configToTransport(t, SecureAnonymousClientConfig(client.ProtoConfig)),
},
{
name: "json config - no cache",
rt: configToTransport(t, bustTLSCache(client.JSONConfig)),
},
{
name: "proto config - no cache",
rt: configToTransport(t, bustTLSCache(client.ProtoConfig)),
},
{
name: "anonymous json config - no cache, inner bust",
rt: configToTransport(t, SecureAnonymousClientConfig(bustTLSCache(client.JSONConfig))),
},
{
name: "anonymous proto config - no cache, inner bust",
rt: configToTransport(t, SecureAnonymousClientConfig(bustTLSCache(client.ProtoConfig))),
},
{
name: "anonymous json config - no cache, double bust",
rt: configToTransport(t, bustTLSCache(SecureAnonymousClientConfig(bustTLSCache(client.JSONConfig)))),
},
{
name: "anonymous proto config - no cache, double bust",
rt: configToTransport(t, bustTLSCache(SecureAnonymousClientConfig(bustTLSCache(client.ProtoConfig)))),
},
{
name: "anonymous json config - no cache, outer bust",
rt: configToTransport(t, bustTLSCache(SecureAnonymousClientConfig(client.JSONConfig))),
},
{
name: "anonymous proto config - no cache, outer bust",
rt: configToTransport(t, bustTLSCache(SecureAnonymousClientConfig(client.ProtoConfig))),
},
}
for _, tt := range tests {
tt := tt
t.Run(tt.name, func(t *testing.T) {
t.Parallel() // make sure to run in parallel to confirm that our client-go TLS cache busting works (i.e. assert no data races)
tlsConfig, err := net.TLSClientConfig(tt.rt)
require.NoError(t, err)
require.NotNil(t, tlsConfig)
secureTLSConfig := ptls.Secure(nil)
require.Equal(t, secureTLSConfig.MinVersion, tlsConfig.MinVersion)
require.Equal(t, secureTLSConfig.CipherSuites, tlsConfig.CipherSuites)
require.Equal(t, secureTLSConfig.NextProtos, tlsConfig.NextProtos)
// x509.CertPool has some embedded functions that make it hard to compare so just look at the subjects
require.Equal(t, serverSubjects, tlsConfig.RootCAs.Subjects())
})
}
}
type restClientGetter interface {
RESTClient() rest.Interface
}
func extractTransport(getter restClientGetter) http.RoundTripper {
return getter.RESTClient().(*rest.RESTClient).Client.Transport
}
func configToTransport(t *testing.T, config *rest.Config) http.RoundTripper {
t.Helper()
rt, err := rest.TransportFor(config)
require.NoError(t, err)
return rt
}
func bustTLSCache(config *rest.Config) *rest.Config {
c := rest.CopyConfig(config)
c.Proxy = func(h *http.Request) (*url.URL, error) {
return nil, nil // having a non-nil proxy func makes client-go not cache the TLS config
}
return c
}
func makeClient(t *testing.T, restConfig *rest.Config, f func(*rest.Config)) *Client {
t.Helper()
restConfig = rest.CopyConfig(restConfig)
f(restConfig)
client, err := New(WithConfig(restConfig))
require.NoError(t, err)
return client
}